projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ixgbevf: add tx counters
[deliverable/linux.git] / drivers / net / ethernet / intel / ixgbevf / ixgbevf_main.c
CommitLineData
92915f71
GR		 1/*******************************************************************************
2
3 Intel 82599 Virtual Function driver
5c47a2b6 4 Copyright(c) 1999 - 2012 Intel Corporation.
92915f71
GR		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28
29/******************************************************************************
30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
31******************************************************************************/
dbd9636e
JK		 32
33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
92915f71 35#include <linux/types.h>
dadcd65f 36#include <linux/bitops.h>
92915f71
GR		 37#include <linux/module.h>
38#include <linux/pci.h>
39#include <linux/netdevice.h>
40#include <linux/vmalloc.h>
41#include <linux/string.h>
42#include <linux/in.h>
43#include <linux/ip.h>
44#include <linux/tcp.h>
70a10e25 45#include <linux/sctp.h>
92915f71 46#include <linux/ipv6.h>
5a0e3ad6 47#include <linux/slab.h>
92915f71
GR		 48#include <net/checksum.h>
49#include <net/ip6_checksum.h>
50#include <linux/ethtool.h>
01789349 51#include <linux/if.h>
92915f71 52#include <linux/if_vlan.h>
70c71606 53#include <linux/prefetch.h>
92915f71
GR		 54
55#include "ixgbevf.h"
56
3d8fe98f 57const char ixgbevf_driver_name[] = "ixgbevf";
92915f71 58static const char ixgbevf_driver_string[] =
422e05d1 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
92915f71 60
86f359f6 61#define DRV_VERSION "2.12.1-k"
92915f71 62const char ixgbevf_driver_version[] = DRV_VERSION;
66c87bd5 63static char ixgbevf_copyright[] =
5c47a2b6 64 "Copyright (c) 2009 - 2012 Intel Corporation.";
92915f71
GR		 65
66static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
2316aa2a
GR		 67 [board_82599_vf] = &ixgbevf_82599_vf_info,
68 [board_X540_vf] = &ixgbevf_X540_vf_info,
92915f71
GR		 69};
70
71/* ixgbevf_pci_tbl - PCI Device ID Table
72 *
73 * Wildcard entries (PCI_ANY_ID) should come last
74 * Last entry must be all 0s
75 *
76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
77 * Class, Class Mask, private data (not used) }
78 */
39ba22b4
SH		 79static DEFINE_PCI_DEVICE_TABLE(ixgbevf_pci_tbl) = {
80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
92915f71
GR		 82 /* required last entry */
83 {0, }
84};
85MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
86
87MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
88MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
89MODULE_LICENSE("GPL");
90MODULE_VERSION(DRV_VERSION);
91
b3f4d599 92#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
93static int debug = -1;
94module_param(debug, int, 0);
95MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
92915f71
GR		 96
97/* forward decls */
220fe050 98static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
fa71ae27 99static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
56e94095 100static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
92915f71 101
5cdab2f6 102static inline void ixgbevf_release_rx_desc(struct ixgbevf_ring *rx_ring,
92915f71
GR		 103 u32 val)
104{
5cdab2f6
DS		 105 rx_ring->next_to_use = val;
106
92915f71
GR		 107 /*
108 * Force memory writes to complete before letting h/w
109 * know there are new descriptors to fetch. (Only
110 * applicable for weak-ordered memory model archs,
111 * such as IA-64).
112 */
113 wmb();
5cdab2f6 114 writel(val, rx_ring->tail);
92915f71
GR		 115}
116
49ce9c2c 117/**
65d676c8 118 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
92915f71
GR		 119 * @adapter: pointer to adapter struct
120 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
121 * @queue: queue to map the corresponding interrupt to
122 * @msix_vector: the vector to map to the corresponding queue
92915f71
GR		 123 */
124static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
125 u8 queue, u8 msix_vector)
126{
127 u32 ivar, index;
128 struct ixgbe_hw *hw = &adapter->hw;
129 if (direction == -1) {
130 /* other causes */
131 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
132 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
133 ivar &= ~0xFF;
134 ivar |= msix_vector;
135 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
136 } else {
137 /* tx or rx causes */
138 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
139 index = ((16 * (queue & 1)) + (8 * direction));
140 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
141 ivar &= ~(0xFF << index);
142 ivar |= (msix_vector << index);
143 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
144 }
145}
146
70a10e25 147static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
92915f71
GR		 148 struct ixgbevf_tx_buffer
149 *tx_buffer_info)
150{
151 if (tx_buffer_info->dma) {
152 if (tx_buffer_info->mapped_as_page)
70a10e25 153 dma_unmap_page(tx_ring->dev,
92915f71
GR		 154 tx_buffer_info->dma,
155 tx_buffer_info->length,
2a1f8794 156 DMA_TO_DEVICE);
92915f71 157 else
70a10e25 158 dma_unmap_single(tx_ring->dev,
92915f71
GR		 159 tx_buffer_info->dma,
160 tx_buffer_info->length,
2a1f8794 161 DMA_TO_DEVICE);
92915f71
GR		 162 tx_buffer_info->dma = 0;
163 }
164 if (tx_buffer_info->skb) {
165 dev_kfree_skb_any(tx_buffer_info->skb);
166 tx_buffer_info->skb = NULL;
167 }
168 tx_buffer_info->time_stamp = 0;
169 /* tx_buffer_info must be completely set up in the transmit path */
170}
171
92915f71
GR		 172#define IXGBE_MAX_TXD_PWR 14
173#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR)
174
175/* Tx Descriptors needed, worst case */
3595990a
AD		 176#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
177#define DESC_NEEDED (MAX_SKB_FRAGS + 4)
92915f71
GR		 178
179static void ixgbevf_tx_timeout(struct net_device *netdev);
180
181/**
182 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
fa71ae27 183 * @q_vector: board private structure
92915f71
GR		 184 * @tx_ring: tx ring to clean
185 **/
fa71ae27 186static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
92915f71
GR		 187 struct ixgbevf_ring *tx_ring)
188{
fa71ae27 189 struct ixgbevf_adapter *adapter = q_vector->adapter;
92915f71
GR		 190 union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
191 struct ixgbevf_tx_buffer *tx_buffer_info;
e757e3e1 192 unsigned int i, count = 0;
92915f71
GR		 193 unsigned int total_bytes = 0, total_packets = 0;
194
10cc1bdd
AD		 195 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
196 return true;
197
92915f71 198 i = tx_ring->next_to_clean;
e757e3e1
AD		 199 tx_buffer_info = &tx_ring->tx_buffer_info[i];
200 eop_desc = tx_buffer_info->next_to_watch;
92915f71 201
e757e3e1 202 do {
92915f71 203 bool cleaned = false;
e757e3e1
AD		 204
205 /* if next_to_watch is not set then there is no work pending */
206 if (!eop_desc)
207 break;
208
209 /* prevent any other reads prior to eop_desc */
210 read_barrier_depends();
211
212 /* if DD is not set pending work has not been completed */
213 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
214 break;
215
216 /* clear next_to_watch to prevent false hangs */
217 tx_buffer_info->next_to_watch = NULL;
218
92915f71
GR		 219 for ( ; !cleaned; count++) {
220 struct sk_buff *skb;
908421f6 221 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
e757e3e1 222 cleaned = (tx_desc == eop_desc);
92915f71
GR		 223 skb = tx_buffer_info->skb;
224
225 if (cleaned && skb) {
226 unsigned int segs, bytecount;
227
228 /* gso_segs is currently only valid for tcp */
229 segs = skb_shinfo(skb)->gso_segs ?: 1;
230 /* multiply data chunks by size of headers */
231 bytecount = ((segs - 1) * skb_headlen(skb)) +
232 skb->len;
233 total_packets += segs;
234 total_bytes += bytecount;
235 }
236
70a10e25 237 ixgbevf_unmap_and_free_tx_resource(tx_ring,
92915f71
GR		 238 tx_buffer_info);
239
240 tx_desc->wb.status = 0;
241
242 i++;
243 if (i == tx_ring->count)
244 i = 0;
e757e3e1
AD		 245
246 tx_buffer_info = &tx_ring->tx_buffer_info[i];
92915f71
GR		 247 }
248
e757e3e1
AD		 249 eop_desc = tx_buffer_info->next_to_watch;
250 } while (count < tx_ring->count);
92915f71
GR		 251
252 tx_ring->next_to_clean = i;
253
254#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
fb40195c 255 if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
f880d07b 256 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
92915f71
GR		 257 /* Make sure that anybody stopping the queue after this
258 * sees the new next_to_clean.
259 */
260 smp_mb();
fb40195c
AD		 261 if (__netif_subqueue_stopped(tx_ring->netdev,
262 tx_ring->queue_index) &&
92915f71 263 !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
fb40195c
AD		 264 netif_wake_subqueue(tx_ring->netdev,
265 tx_ring->queue_index);
92915f71
GR		 266 ++adapter->restart_queue;
267 }
92915f71
GR		 268 }
269
4197aa7b 270 u64_stats_update_begin(&tx_ring->syncp);
095e2617
ET		 271 tx_ring->stats.bytes += total_bytes;
272 tx_ring->stats.packets += total_packets;
4197aa7b 273 u64_stats_update_end(&tx_ring->syncp);
ac6ed8f0
GR		 274 q_vector->tx.total_bytes += total_bytes;
275 q_vector->tx.total_packets += total_packets;
92915f71 276
fa71ae27 277 return count < tx_ring->count;
92915f71
GR		 278}
279
280/**
281 * ixgbevf_receive_skb - Send a completed packet up the stack
282 * @q_vector: structure containing interrupt and ring information
283 * @skb: packet to send up
284 * @status: hardware indication of status of receive
92915f71
GR		 285 * @rx_desc: rx descriptor
286 **/
287static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
288 struct sk_buff *skb, u8 status,
92915f71
GR		 289 union ixgbe_adv_rx_desc *rx_desc)
290{
291 struct ixgbevf_adapter *adapter = q_vector->adapter;
292 bool is_vlan = (status & IXGBE_RXD_STAT_VP);
dd1ed3b7 293 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
92915f71 294
5d9a533b 295 if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
86a9bad3 296 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
dadcd65f 297
366c1099
GR		 298 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
299 napi_gro_receive(&q_vector->napi, skb);
300 else
301 netif_rx(skb);
92915f71
GR		 302}
303
08681618
JK		 304/**
305 * ixgbevf_rx_skb - Helper function to determine proper Rx method
306 * @q_vector: structure containing interrupt and ring information
307 * @skb: packet to send up
308 * @status: hardware indication of status of receive
309 * @rx_desc: rx descriptor
310 **/
311static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
312 struct sk_buff *skb, u8 status,
313 union ixgbe_adv_rx_desc *rx_desc)
314{
c777cdfa
JK		 315#ifdef CONFIG_NET_RX_BUSY_POLL
316 skb_mark_napi_id(skb, &q_vector->napi);
317
318 if (ixgbevf_qv_busy_polling(q_vector)) {
319 netif_receive_skb(skb);
320 /* exit early if we busy polled */
321 return;
322 }
323#endif /* CONFIG_NET_RX_BUSY_POLL */
324
08681618
JK		 325 ixgbevf_receive_skb(q_vector, skb, status, rx_desc);
326}
327
92915f71
GR		 328/**
329 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
55fb277c 330 * @ring: pointer to Rx descriptor ring structure
92915f71
GR		 331 * @status_err: hardware indication of status of receive
332 * @skb: skb currently being received and modified
333 **/
55fb277c 334static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
92915f71
GR		 335 u32 status_err, struct sk_buff *skb)
336{
bc8acf2c 337 skb_checksum_none_assert(skb);
92915f71
GR		 338
339 /* Rx csum disabled */
fb40195c 340 if (!(ring->netdev->features & NETIF_F_RXCSUM))
92915f71
GR		 341 return;
342
343 /* if IP and error */
344 if ((status_err & IXGBE_RXD_STAT_IPCS) &&
345 (status_err & IXGBE_RXDADV_ERR_IPE)) {
095e2617 346 ring->rx_stats.csum_err++;
92915f71
GR		 347 return;
348 }
349
350 if (!(status_err & IXGBE_RXD_STAT_L4CS))
351 return;
352
353 if (status_err & IXGBE_RXDADV_ERR_TCPE) {
095e2617 354 ring->rx_stats.csum_err++;
92915f71
GR		 355 return;
356 }
357
358 /* It must be a TCP or UDP packet with a valid checksum */
359 skb->ip_summed = CHECKSUM_UNNECESSARY;
92915f71
GR		 360}
361
362/**
363 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
095e2617 364 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
92915f71 365 **/
095e2617 366static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
92915f71
GR		 367 int cleaned_count)
368{
92915f71
GR		 369 union ixgbe_adv_rx_desc *rx_desc;
370 struct ixgbevf_rx_buffer *bi;
fb40195c 371 unsigned int i = rx_ring->next_to_use;
92915f71 372
92915f71 373 while (cleaned_count--) {
908421f6 374 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
05d063aa 375 bi = &rx_ring->rx_buffer_info[i];
b9dd245b
GR		 376
377 if (!bi->skb) {
378 struct sk_buff *skb;
379
fb40195c
AD		 380 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
381 rx_ring->rx_buf_len);
05d063aa 382 if (!skb)
92915f71 383 goto no_buffers;
05d063aa 384
92915f71 385 bi->skb = skb;
b9dd245b 386
05d063aa 387 bi->dma = dma_map_single(rx_ring->dev, skb->data,
92915f71 388 rx_ring->rx_buf_len,
2a1f8794 389 DMA_FROM_DEVICE);
05d063aa 390 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
6132ee8a
GR		 391 dev_kfree_skb(skb);
392 bi->skb = NULL;
05d063aa 393 dev_err(rx_ring->dev, "Rx DMA map failed\n");
6132ee8a
GR		 394 break;
395 }
92915f71 396 }
77d5dfca 397 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
92915f71
GR		 398
399 i++;
400 if (i == rx_ring->count)
401 i = 0;
92915f71
GR		 402 }
403
404no_buffers:
095e2617 405 rx_ring->rx_stats.alloc_rx_buff_failed++;
5cdab2f6
DS		 406 if (rx_ring->next_to_use != i)
407 ixgbevf_release_rx_desc(rx_ring, i);
92915f71
GR		 408}
409
410static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
5f3600eb 411 u32 qmask)
92915f71 412{
92915f71
GR		 413 struct ixgbe_hw *hw = &adapter->hw;
414
5f3600eb 415 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
92915f71
GR		 416}
417
08e50a20
JK		 418static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
419 struct ixgbevf_ring *rx_ring,
420 int budget)
92915f71 421{
92915f71
GR		 422 union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
423 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
424 struct sk_buff *skb;
425 unsigned int i;
426 u32 len, staterr;
92915f71
GR		 427 int cleaned_count = 0;
428 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
429
430 i = rx_ring->next_to_clean;
908421f6 431 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 432 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
433 rx_buffer_info = &rx_ring->rx_buffer_info[i];
434
435 while (staterr & IXGBE_RXD_STAT_DD) {
fa71ae27 436 if (!budget)
92915f71 437 break;
fa71ae27 438 budget--;
92915f71 439
2d0bb1c1 440 rmb(); /* read descriptor and rx_buffer_info after status DD */
77d5dfca 441 len = le16_to_cpu(rx_desc->wb.upper.length);
92915f71
GR		 442 skb = rx_buffer_info->skb;
443 prefetch(skb->data - NET_IP_ALIGN);
444 rx_buffer_info->skb = NULL;
445
446 if (rx_buffer_info->dma) {
05d063aa 447 dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
92915f71 448 rx_ring->rx_buf_len,
2a1f8794 449 DMA_FROM_DEVICE);
92915f71
GR		 450 rx_buffer_info->dma = 0;
451 skb_put(skb, len);
452 }
453
92915f71
GR		 454 i++;
455 if (i == rx_ring->count)
456 i = 0;
457
908421f6 458 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 459 prefetch(next_rxd);
460 cleaned_count++;
461
462 next_buffer = &rx_ring->rx_buffer_info[i];
463
464 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
77d5dfca 465 skb->next = next_buffer->skb;
5c60f81a 466 IXGBE_CB(skb->next)->prev = skb;
095e2617 467 rx_ring->rx_stats.non_eop_descs++;
92915f71
GR		 468 goto next_desc;
469 }
470
5c60f81a
AD		 471 /* we should not be chaining buffers, if we did drop the skb */
472 if (IXGBE_CB(skb)->prev) {
473 do {
474 struct sk_buff *this = skb;
475 skb = IXGBE_CB(skb)->prev;
476 dev_kfree_skb(this);
477 } while (skb);
478 goto next_desc;
479 }
480
92915f71
GR		 481 /* ERR_MASK will only have valid bits if EOP set */
482 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
483 dev_kfree_skb_irq(skb);
484 goto next_desc;
485 }
486
55fb277c 487 ixgbevf_rx_checksum(rx_ring, staterr, skb);
92915f71
GR		 488
489 /* probably a little skewed due to removing CRC */
490 total_rx_bytes += skb->len;
491 total_rx_packets++;
492
fb40195c 493 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
92915f71 494
815cccbf
JF		 495 /* Workaround hardware that can't do proper VEPA multicast
496 * source pruning.
497 */
498 if ((skb->pkt_type & (PACKET_BROADCAST | PACKET_MULTICAST)) &&
095e2617 499 ether_addr_equal(rx_ring->netdev->dev_addr,
7367d0b5 500 eth_hdr(skb)->h_source)) {
815cccbf
JF		 501 dev_kfree_skb_irq(skb);
502 goto next_desc;
503 }
504
08681618 505 ixgbevf_rx_skb(q_vector, skb, staterr, rx_desc);
92915f71
GR		 506
507next_desc:
508 rx_desc->wb.upper.status_error = 0;
509
510 /* return some buffers to hardware, one at a time is too slow */
511 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
095e2617 512 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
92915f71
GR		 513 cleaned_count = 0;
514 }
515
516 /* use prefetched values */
517 rx_desc = next_rxd;
518 rx_buffer_info = &rx_ring->rx_buffer_info[i];
519
520 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
521 }
522
523 rx_ring->next_to_clean = i;
f880d07b 524 cleaned_count = ixgbevf_desc_unused(rx_ring);
92915f71
GR		 525
526 if (cleaned_count)
095e2617 527 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
92915f71 528
4197aa7b 529 u64_stats_update_begin(&rx_ring->syncp);
095e2617
ET		 530 rx_ring->stats.packets += total_rx_packets;
531 rx_ring->stats.bytes += total_rx_bytes;
4197aa7b 532 u64_stats_update_end(&rx_ring->syncp);
ac6ed8f0
GR		 533 q_vector->rx.total_packets += total_rx_packets;
534 q_vector->rx.total_bytes += total_rx_bytes;
92915f71 535
08e50a20 536 return total_rx_packets;
92915f71
GR		 537}
538
539/**
fa71ae27 540 * ixgbevf_poll - NAPI polling calback
92915f71
GR		 541 * @napi: napi struct with our devices info in it
542 * @budget: amount of work driver is allowed to do this pass, in packets
543 *
fa71ae27 544 * This function will clean more than one or more rings associated with a
92915f71
GR		 545 * q_vector.
546 **/
fa71ae27 547static int ixgbevf_poll(struct napi_struct *napi, int budget)
92915f71
GR		 548{
549 struct ixgbevf_q_vector *q_vector =
550 container_of(napi, struct ixgbevf_q_vector, napi);
551 struct ixgbevf_adapter *adapter = q_vector->adapter;
fa71ae27
AD		 552 struct ixgbevf_ring *ring;
553 int per_ring_budget;
554 bool clean_complete = true;
555
556 ixgbevf_for_each_ring(ring, q_vector->tx)
557 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
92915f71 558
c777cdfa
JK		 559#ifdef CONFIG_NET_RX_BUSY_POLL
560 if (!ixgbevf_qv_lock_napi(q_vector))
561 return budget;
562#endif
563
92915f71
GR		 564 /* attempt to distribute budget to each queue fairly, but don't allow
565 * the budget to go below 1 because we'll exit polling */
fa71ae27
AD		 566 if (q_vector->rx.count > 1)
567 per_ring_budget = max(budget/q_vector->rx.count, 1);
568 else
569 per_ring_budget = budget;
570
366c1099 571 adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
fa71ae27 572 ixgbevf_for_each_ring(ring, q_vector->rx)
08e50a20
JK		 573 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring,
574 per_ring_budget)
575 < per_ring_budget);
366c1099 576 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
fa71ae27 577
c777cdfa
JK		 578#ifdef CONFIG_NET_RX_BUSY_POLL
579 ixgbevf_qv_unlock_napi(q_vector);
580#endif
581
fa71ae27
AD		 582 /* If all work not completed, return budget and keep polling */
583 if (!clean_complete)
584 return budget;
585 /* all work done, exit the polling mode */
586 napi_complete(napi);
587 if (adapter->rx_itr_setting & 1)
588 ixgbevf_set_itr(q_vector);
589 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
590 ixgbevf_irq_enable_queues(adapter,
591 1 << q_vector->v_idx);
92915f71 592
fa71ae27 593 return 0;
92915f71
GR		 594}
595
ce422606
GR		 596/**
597 * ixgbevf_write_eitr - write VTEITR register in hardware specific way
598 * @q_vector: structure containing interrupt and ring information
599 */
3849623e 600void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
ce422606
GR		 601{
602 struct ixgbevf_adapter *adapter = q_vector->adapter;
603 struct ixgbe_hw *hw = &adapter->hw;
604 int v_idx = q_vector->v_idx;
605 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
606
607 /*
608 * set the WDIS bit to not clear the timer bits and cause an
609 * immediate assertion of the interrupt
610 */
611 itr_reg |= IXGBE_EITR_CNT_WDIS;
612
613 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
614}
92915f71 615
c777cdfa
JK		 616#ifdef CONFIG_NET_RX_BUSY_POLL
617/* must be called with local_bh_disable()d */
618static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
619{
620 struct ixgbevf_q_vector *q_vector =
621 container_of(napi, struct ixgbevf_q_vector, napi);
622 struct ixgbevf_adapter *adapter = q_vector->adapter;
623 struct ixgbevf_ring *ring;
624 int found = 0;
625
626 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
627 return LL_FLUSH_FAILED;
628
629 if (!ixgbevf_qv_lock_poll(q_vector))
630 return LL_FLUSH_BUSY;
631
632 ixgbevf_for_each_ring(ring, q_vector->rx) {
633 found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
3b5dca26
JK		 634#ifdef BP_EXTENDED_STATS
635 if (found)
095e2617 636 ring->stats.cleaned += found;
3b5dca26 637 else
095e2617 638 ring->stats.misses++;
3b5dca26 639#endif
c777cdfa
JK		 640 if (found)
641 break;
642 }
643
644 ixgbevf_qv_unlock_poll(q_vector);
645
646 return found;
647}
648#endif /* CONFIG_NET_RX_BUSY_POLL */
649
92915f71
GR		 650/**
651 * ixgbevf_configure_msix - Configure MSI-X hardware
652 * @adapter: board private structure
653 *
654 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
655 * interrupts.
656 **/
657static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
658{
659 struct ixgbevf_q_vector *q_vector;
6b43c446 660 int q_vectors, v_idx;
92915f71
GR		 661
662 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
5f3600eb 663 adapter->eims_enable_mask = 0;
92915f71
GR		 664
665 /*
666 * Populate the IVAR table and set the ITR values to the
667 * corresponding register.
668 */
669 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
6b43c446 670 struct ixgbevf_ring *ring;
92915f71 671 q_vector = adapter->q_vector[v_idx];
6b43c446
AD		 672
673 ixgbevf_for_each_ring(ring, q_vector->rx)
674 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
675
676 ixgbevf_for_each_ring(ring, q_vector->tx)
677 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
92915f71 678
5f3600eb
AD		 679 if (q_vector->tx.ring && !q_vector->rx.ring) {
680 /* tx only vector */
681 if (adapter->tx_itr_setting == 1)
682 q_vector->itr = IXGBE_10K_ITR;
683 else
684 q_vector->itr = adapter->tx_itr_setting;
685 } else {
686 /* rx or rx/tx vector */
687 if (adapter->rx_itr_setting == 1)
688 q_vector->itr = IXGBE_20K_ITR;
689 else
690 q_vector->itr = adapter->rx_itr_setting;
691 }
692
693 /* add q_vector eims value to global eims_enable_mask */
694 adapter->eims_enable_mask |= 1 << v_idx;
92915f71 695
5f3600eb 696 ixgbevf_write_eitr(q_vector);
92915f71
GR		 697 }
698
699 ixgbevf_set_ivar(adapter, -1, 1, v_idx);
5f3600eb
AD		 700 /* setup eims_other and add value to global eims_enable_mask */
701 adapter->eims_other = 1 << v_idx;
702 adapter->eims_enable_mask |= adapter->eims_other;
92915f71
GR		 703}
704
705enum latency_range {
706 lowest_latency = 0,
707 low_latency = 1,
708 bulk_latency = 2,
709 latency_invalid = 255
710};
711
712/**
713 * ixgbevf_update_itr - update the dynamic ITR value based on statistics
5f3600eb
AD		 714 * @q_vector: structure containing interrupt and ring information
715 * @ring_container: structure containing ring performance data
92915f71
GR		 716 *
717 * Stores a new ITR value based on packets and byte
718 * counts during the last interrupt. The advantage of per interrupt
719 * computation is faster updates and more accurate ITR for the current
720 * traffic pattern. Constants in this function were computed
721 * based on theoretical maximum wire speed and thresholds were set based
722 * on testing data as well as attempting to minimize response time
723 * while increasing bulk throughput.
724 **/
5f3600eb
AD		 725static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
726 struct ixgbevf_ring_container *ring_container)
92915f71 727{
5f3600eb
AD		 728 int bytes = ring_container->total_bytes;
729 int packets = ring_container->total_packets;
92915f71
GR		 730 u32 timepassed_us;
731 u64 bytes_perint;
5f3600eb 732 u8 itr_setting = ring_container->itr;
92915f71
GR		 733
734 if (packets == 0)
5f3600eb 735 return;
92915f71
GR		 736
737 /* simple throttlerate management
738 * 0-20MB/s lowest (100000 ints/s)
739 * 20-100MB/s low (20000 ints/s)
740 * 100-1249MB/s bulk (8000 ints/s)
741 */
742 /* what was last interrupt timeslice? */
5f3600eb 743 timepassed_us = q_vector->itr >> 2;
92915f71
GR		 744 bytes_perint = bytes / timepassed_us; /* bytes/usec */
745
746 switch (itr_setting) {
747 case lowest_latency:
e2c28ce7 748 if (bytes_perint > 10)
5f3600eb 749 itr_setting = low_latency;
92915f71
GR		 750 break;
751 case low_latency:
e2c28ce7 752 if (bytes_perint > 20)
5f3600eb 753 itr_setting = bulk_latency;
e2c28ce7 754 else if (bytes_perint <= 10)
5f3600eb 755 itr_setting = lowest_latency;
92915f71
GR		 756 break;
757 case bulk_latency:
e2c28ce7 758 if (bytes_perint <= 20)
5f3600eb 759 itr_setting = low_latency;
92915f71
GR		 760 break;
761 }
762
5f3600eb
AD		 763 /* clear work counters since we have the values we need */
764 ring_container->total_bytes = 0;
765 ring_container->total_packets = 0;
766
767 /* write updated itr to ring container */
768 ring_container->itr = itr_setting;
92915f71
GR		 769}
770
fa71ae27 771static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
92915f71 772{
5f3600eb
AD		 773 u32 new_itr = q_vector->itr;
774 u8 current_itr;
92915f71 775
5f3600eb
AD		 776 ixgbevf_update_itr(q_vector, &q_vector->tx);
777 ixgbevf_update_itr(q_vector, &q_vector->rx);
92915f71 778
6b43c446 779 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
92915f71
GR		 780
781 switch (current_itr) {
782 /* counts and packets in update_itr are dependent on these numbers */
783 case lowest_latency:
5f3600eb 784 new_itr = IXGBE_100K_ITR;
92915f71
GR		 785 break;
786 case low_latency:
5f3600eb 787 new_itr = IXGBE_20K_ITR;
92915f71
GR		 788 break;
789 case bulk_latency:
790 default:
5f3600eb 791 new_itr = IXGBE_8K_ITR;
92915f71
GR		 792 break;
793 }
794
5f3600eb 795 if (new_itr != q_vector->itr) {
92915f71 796 /* do an exponential smoothing */
5f3600eb
AD		 797 new_itr = (10 * new_itr * q_vector->itr) /
798 ((9 * new_itr) + q_vector->itr);
799
800 /* save the algorithm value here */
801 q_vector->itr = new_itr;
802
803 ixgbevf_write_eitr(q_vector);
92915f71 804 }
92915f71
GR		 805}
806
4b2cd27f 807static irqreturn_t ixgbevf_msix_other(int irq, void *data)
92915f71 808{
fa71ae27 809 struct ixgbevf_adapter *adapter = data;
92915f71 810 struct ixgbe_hw *hw = &adapter->hw;
08259594 811
4b2cd27f 812 hw->mac.get_link_status = 1;
1e72bfc3 813
c7bb417d
DS		 814 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
815 mod_timer(&adapter->watchdog_timer, jiffies);
3a2c4033 816
5f3600eb
AD		 817 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
818
92915f71
GR		 819 return IRQ_HANDLED;
820}
821
92915f71 822/**
fa71ae27 823 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
92915f71
GR		 824 * @irq: unused
825 * @data: pointer to our q_vector struct for this interrupt vector
826 **/
fa71ae27 827static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
92915f71
GR		 828{
829 struct ixgbevf_q_vector *q_vector = data;
92915f71 830
5f3600eb 831 /* EIAM disabled interrupts (on this vector) for us */
fa71ae27
AD		 832 if (q_vector->rx.ring || q_vector->tx.ring)
833 napi_schedule(&q_vector->napi);
92915f71
GR		 834
835 return IRQ_HANDLED;
836}
837
838static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
839 int r_idx)
840{
841 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
842
87e70ab9
DS		 843 a->rx_ring[r_idx]->next = q_vector->rx.ring;
844 q_vector->rx.ring = a->rx_ring[r_idx];
6b43c446 845 q_vector->rx.count++;
92915f71
GR		 846}
847
848static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
849 int t_idx)
850{
851 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
852
87e70ab9
DS		 853 a->tx_ring[t_idx]->next = q_vector->tx.ring;
854 q_vector->tx.ring = a->tx_ring[t_idx];
6b43c446 855 q_vector->tx.count++;
92915f71
GR		 856}
857
858/**
859 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
860 * @adapter: board private structure to initialize
861 *
862 * This function maps descriptor rings to the queue-specific vectors
863 * we were allotted through the MSI-X enabling code. Ideally, we'd have
864 * one vector per ring/queue, but on a constrained vector budget, we
865 * group the rings as "efficiently" as possible. You would add new
866 * mapping configurations in here.
867 **/
868static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
869{
870 int q_vectors;
871 int v_start = 0;
872 int rxr_idx = 0, txr_idx = 0;
873 int rxr_remaining = adapter->num_rx_queues;
874 int txr_remaining = adapter->num_tx_queues;
875 int i, j;
876 int rqpv, tqpv;
877 int err = 0;
878
879 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
880
881 /*
882 * The ideal configuration...
883 * We have enough vectors to map one per queue.
884 */
885 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
886 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
887 map_vector_to_rxq(adapter, v_start, rxr_idx);
888
889 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
890 map_vector_to_txq(adapter, v_start, txr_idx);
891 goto out;
892 }
893
894 /*
895 * If we don't have enough vectors for a 1-to-1
896 * mapping, we'll have to group them so there are
897 * multiple queues per vector.
898 */
899 /* Re-adjusting *qpv takes care of the remainder. */
900 for (i = v_start; i < q_vectors; i++) {
901 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
902 for (j = 0; j < rqpv; j++) {
903 map_vector_to_rxq(adapter, i, rxr_idx);
904 rxr_idx++;
905 rxr_remaining--;
906 }
907 }
908 for (i = v_start; i < q_vectors; i++) {
909 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
910 for (j = 0; j < tqpv; j++) {
911 map_vector_to_txq(adapter, i, txr_idx);
912 txr_idx++;
913 txr_remaining--;
914 }
915 }
916
917out:
918 return err;
919}
920
921/**
922 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
923 * @adapter: board private structure
924 *
925 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
926 * interrupts from the kernel.
927 **/
928static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
929{
930 struct net_device *netdev = adapter->netdev;
fa71ae27
AD		 931 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
932 int vector, err;
92915f71
GR		 933 int ri = 0, ti = 0;
934
92915f71 935 for (vector = 0; vector < q_vectors; vector++) {
fa71ae27
AD		 936 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
937 struct msix_entry *entry = &adapter->msix_entries[vector];
938
939 if (q_vector->tx.ring && q_vector->rx.ring) {
940 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
941 "%s-%s-%d", netdev->name, "TxRx", ri++);
942 ti++;
943 } else if (q_vector->rx.ring) {
944 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
945 "%s-%s-%d", netdev->name, "rx", ri++);
946 } else if (q_vector->tx.ring) {
947 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
948 "%s-%s-%d", netdev->name, "tx", ti++);
92915f71
GR		 949 } else {
950 /* skip this unused q_vector */
951 continue;
952 }
fa71ae27
AD		 953 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
954 q_vector->name, q_vector);
92915f71
GR		 955 if (err) {
956 hw_dbg(&adapter->hw,
957 "request_irq failed for MSIX interrupt "
958 "Error: %d\n", err);
959 goto free_queue_irqs;
960 }
961 }
962
92915f71 963 err = request_irq(adapter->msix_entries[vector].vector,
4b2cd27f 964 &ixgbevf_msix_other, 0, netdev->name, adapter);
92915f71
GR		 965 if (err) {
966 hw_dbg(&adapter->hw,
4b2cd27f 967 "request_irq for msix_other failed: %d\n", err);
92915f71
GR		 968 goto free_queue_irqs;
969 }
970
971 return 0;
972
973free_queue_irqs:
fa71ae27
AD		 974 while (vector) {
975 vector--;
976 free_irq(adapter->msix_entries[vector].vector,
977 adapter->q_vector[vector]);
978 }
a1f6c6b1 979 /* This failure is non-recoverable - it indicates the system is
980 * out of MSIX vector resources and the VF driver cannot run
981 * without them. Set the number of msix vectors to zero
982 * indicating that not enough can be allocated. The error
983 * will be returned to the user indicating device open failed.
984 * Any further attempts to force the driver to open will also
985 * fail. The only way to recover is to unload the driver and
986 * reload it again. If the system has recovered some MSIX
987 * vectors then it may succeed.
988 */
989 adapter->num_msix_vectors = 0;
92915f71
GR		 990 return err;
991}
992
993static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
994{
995 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
996
997 for (i = 0; i < q_vectors; i++) {
998 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
6b43c446
AD		 999 q_vector->rx.ring = NULL;
1000 q_vector->tx.ring = NULL;
1001 q_vector->rx.count = 0;
1002 q_vector->tx.count = 0;
92915f71
GR		 1003 }
1004}
1005
1006/**
1007 * ixgbevf_request_irq - initialize interrupts
1008 * @adapter: board private structure
1009 *
1010 * Attempts to configure interrupts using the best available
1011 * capabilities of the hardware and kernel.
1012 **/
1013static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1014{
1015 int err = 0;
1016
1017 err = ixgbevf_request_msix_irqs(adapter);
1018
1019 if (err)
1020 hw_dbg(&adapter->hw,
1021 "request_irq failed, Error %d\n", err);
1022
1023 return err;
1024}
1025
1026static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1027{
92915f71
GR		 1028 int i, q_vectors;
1029
1030 q_vectors = adapter->num_msix_vectors;
92915f71
GR		 1031 i = q_vectors - 1;
1032
fa71ae27 1033 free_irq(adapter->msix_entries[i].vector, adapter);
92915f71
GR		 1034 i--;
1035
1036 for (; i >= 0; i--) {
fa71ae27
AD		 1037 /* free only the irqs that were actually requested */
1038 if (!adapter->q_vector[i]->rx.ring &&
1039 !adapter->q_vector[i]->tx.ring)
1040 continue;
1041
92915f71
GR		 1042 free_irq(adapter->msix_entries[i].vector,
1043 adapter->q_vector[i]);
1044 }
1045
1046 ixgbevf_reset_q_vectors(adapter);
1047}
1048
1049/**
1050 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1051 * @adapter: board private structure
1052 **/
1053static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1054{
92915f71 1055 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 1056 int i;
92915f71 1057
5f3600eb 1058 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
92915f71 1059 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
5f3600eb 1060 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
92915f71
GR		 1061
1062 IXGBE_WRITE_FLUSH(hw);
1063
1064 for (i = 0; i < adapter->num_msix_vectors; i++)
1065 synchronize_irq(adapter->msix_entries[i].vector);
1066}
1067
1068/**
1069 * ixgbevf_irq_enable - Enable default interrupt generation settings
1070 * @adapter: board private structure
1071 **/
5f3600eb 1072static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
92915f71
GR		 1073{
1074 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1075
5f3600eb
AD		 1076 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1077 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1078 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
92915f71
GR		 1079}
1080
de02decb
DS		 1081/**
1082 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1083 * @adapter: board private structure
1084 * @ring: structure containing ring specific data
1085 *
1086 * Configure the Tx descriptor ring after a reset.
1087 **/
1088static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1089 struct ixgbevf_ring *ring)
1090{
1091 struct ixgbe_hw *hw = &adapter->hw;
1092 u64 tdba = ring->dma;
1093 int wait_loop = 10;
1094 u32 txdctl = IXGBE_TXDCTL_ENABLE;
1095 u8 reg_idx = ring->reg_idx;
1096
1097 /* disable queue to avoid issues while updating state */
1098 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1099 IXGBE_WRITE_FLUSH(hw);
1100
1101 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1102 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1103 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1104 ring->count * sizeof(union ixgbe_adv_tx_desc));
1105
1106 /* disable head writeback */
1107 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1108 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1109
1110 /* enable relaxed ordering */
1111 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1112 (IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1113 IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1114
1115 /* reset head and tail pointers */
1116 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1117 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
1118 ring->tail = hw->hw_addr + IXGBE_VFTDT(reg_idx);
1119
1120 /* reset ntu and ntc to place SW in sync with hardwdare */
1121 ring->next_to_clean = 0;
1122 ring->next_to_use = 0;
1123
1124 /* In order to avoid issues WTHRESH + PTHRESH should always be equal
1125 * to or less than the number of on chip descriptors, which is
1126 * currently 40.
1127 */
1128 txdctl |= (8 << 16); /* WTHRESH = 8 */
1129
1130 /* Setting PTHRESH to 32 both improves performance */
1131 txdctl |= (1 << 8) | /* HTHRESH = 1 */
1132 32; /* PTHRESH = 32 */
1133
1134 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1135
1136 /* poll to verify queue is enabled */
1137 do {
1138 usleep_range(1000, 2000);
1139 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1140 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1141 if (!wait_loop)
1142 pr_err("Could not enable Tx Queue %d\n", reg_idx);
1143}
1144
92915f71
GR		 1145/**
1146 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1147 * @adapter: board private structure
1148 *
1149 * Configure the Tx unit of the MAC after a reset.
1150 **/
1151static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1152{
de02decb 1153 u32 i;
92915f71
GR		 1154
1155 /* Setup the HW Tx Head and Tail descriptor pointers */
de02decb
DS		 1156 for (i = 0; i < adapter->num_tx_queues; i++)
1157 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);
92915f71
GR		 1158}
1159
1160#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1161
1162static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1163{
1164 struct ixgbevf_ring *rx_ring;
1165 struct ixgbe_hw *hw = &adapter->hw;
1166 u32 srrctl;
1167
87e70ab9 1168 rx_ring = adapter->rx_ring[index];
92915f71
GR		 1169
1170 srrctl = IXGBE_SRRCTL_DROP_EN;
1171
77d5dfca 1172 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
92915f71 1173
dd1fe113
AD		 1174 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1175 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1176
92915f71
GR		 1177 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1178}
1179
1bb9c639
DS		 1180static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1181{
1182 struct ixgbe_hw *hw = &adapter->hw;
1183
1184 /* PSRTYPE must be initialized in 82599 */
1185 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1186 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1187 IXGBE_PSRTYPE_L2HDR;
1188
1189 if (adapter->num_rx_queues > 1)
1190 psrtype |= 1 << 29;
1191
1192 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1193}
1194
dd1fe113
AD		 1195static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1196{
1197 struct ixgbe_hw *hw = &adapter->hw;
1198 struct net_device *netdev = adapter->netdev;
1199 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1200 int i;
1201 u16 rx_buf_len;
1202
1203 /* notify the PF of our intent to use this size of frame */
1204 ixgbevf_rlpml_set_vf(hw, max_frame);
1205
1206 /* PF will allow an extra 4 bytes past for vlan tagged frames */
1207 max_frame += VLAN_HLEN;
1208
1209 /*
85624caf
GR		 1210 * Allocate buffer sizes that fit well into 32K and
1211 * take into account max frame size of 9.5K
dd1fe113
AD		 1212 */
1213 if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1214 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1215 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85624caf
GR		 1216 else if (max_frame <= IXGBEVF_RXBUFFER_2K)
1217 rx_buf_len = IXGBEVF_RXBUFFER_2K;
1218 else if (max_frame <= IXGBEVF_RXBUFFER_4K)
1219 rx_buf_len = IXGBEVF_RXBUFFER_4K;
1220 else if (max_frame <= IXGBEVF_RXBUFFER_8K)
1221 rx_buf_len = IXGBEVF_RXBUFFER_8K;
dd1fe113 1222 else
85624caf 1223 rx_buf_len = IXGBEVF_RXBUFFER_10K;
dd1fe113
AD		 1224
1225 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 1226 adapter->rx_ring[i]->rx_buf_len = rx_buf_len;
dd1fe113
AD		 1227}
1228
de02decb
DS		 1229#define IXGBEVF_MAX_RX_DESC_POLL 10
1230static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1231 struct ixgbevf_ring *ring)
1232{
1233 struct ixgbe_hw *hw = &adapter->hw;
1234 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1235 u32 rxdctl;
1236 u8 reg_idx = ring->reg_idx;
1237
1238 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1239 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1240
1241 /* write value back with RXDCTL.ENABLE bit cleared */
1242 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1243
1244 /* the hardware may take up to 100us to really disable the rx queue */
1245 do {
1246 udelay(10);
1247 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1248 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1249
1250 if (!wait_loop)
1251 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1252 reg_idx);
1253}
1254
1255static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1256 struct ixgbevf_ring *ring)
1257{
1258 struct ixgbe_hw *hw = &adapter->hw;
1259 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1260 u32 rxdctl;
1261 u8 reg_idx = ring->reg_idx;
1262
1263 do {
1264 usleep_range(1000, 2000);
1265 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1266 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1267
1268 if (!wait_loop)
1269 pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1270 reg_idx);
1271}
1272
1273static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1274 struct ixgbevf_ring *ring)
1275{
1276 struct ixgbe_hw *hw = &adapter->hw;
1277 u64 rdba = ring->dma;
1278 u32 rxdctl;
1279 u8 reg_idx = ring->reg_idx;
1280
1281 /* disable queue to avoid issues while updating state */
1282 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1283 ixgbevf_disable_rx_queue(adapter, ring);
1284
1285 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1286 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1287 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1288 ring->count * sizeof(union ixgbe_adv_rx_desc));
1289
1290 /* enable relaxed ordering */
1291 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1292 IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1293
1294 /* reset head and tail pointers */
1295 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1296 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
1297 ring->tail = hw->hw_addr + IXGBE_VFRDT(reg_idx);
1298
1299 /* reset ntu and ntc to place SW in sync with hardwdare */
1300 ring->next_to_clean = 0;
1301 ring->next_to_use = 0;
1302
1303 ixgbevf_configure_srrctl(adapter, reg_idx);
1304
1305 /* prevent DMA from exceeding buffer space available */
1306 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1307 rxdctl |= ring->rx_buf_len | IXGBE_RXDCTL_RLPML_EN;
1308 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1309 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1310
1311 ixgbevf_rx_desc_queue_enable(adapter, ring);
095e2617 1312 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
de02decb
DS		 1313}
1314
92915f71
GR		 1315/**
1316 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1317 * @adapter: board private structure
1318 *
1319 * Configure the Rx unit of the MAC after a reset.
1320 **/
1321static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1322{
de02decb 1323 int i;
92915f71 1324
1bb9c639 1325 ixgbevf_setup_psrtype(adapter);
dd1fe113
AD		 1326
1327 /* set_rx_buffer_len must be called before ring initialization */
1328 ixgbevf_set_rx_buffer_len(adapter);
92915f71 1329
92915f71
GR		 1330 /* Setup the HW Rx Head and Tail Descriptor Pointers and
1331 * the Base and Length of the Rx Descriptor Ring */
de02decb
DS		 1332 for (i = 0; i < adapter->num_rx_queues; i++)
1333 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]);
92915f71
GR		 1334}
1335
80d5c368
PM		 1336static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
1337 __be16 proto, u16 vid)
92915f71
GR		 1338{
1339 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1340 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1
AD		 1341 int err;
1342
55fdd45b 1343 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1344
92915f71 1345 /* add VID to filter table */
2ddc7fe1 1346 err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1c55ed76 1347
55fdd45b 1348 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1349
2ddc7fe1
AD		 1350 /* translate error return types so error makes sense */
1351 if (err == IXGBE_ERR_MBX)
1352 return -EIO;
1353
1354 if (err == IXGBE_ERR_INVALID_ARGUMENT)
1355 return -EACCES;
1356
dadcd65f 1357 set_bit(vid, adapter->active_vlans);
8e586137 1358
2ddc7fe1 1359 return err;
92915f71
GR		 1360}
1361
80d5c368
PM		 1362static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
1363 __be16 proto, u16 vid)
92915f71
GR		 1364{
1365 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1366 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1 1367 int err = -EOPNOTSUPP;
92915f71 1368
55fdd45b 1369 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1370
92915f71 1371 /* remove VID from filter table */
92fe0bf7 1372 err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1c55ed76 1373
55fdd45b 1374 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1375
dadcd65f 1376 clear_bit(vid, adapter->active_vlans);
8e586137 1377
2ddc7fe1 1378 return err;
92915f71
GR		 1379}
1380
1381static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1382{
dadcd65f 1383 u16 vid;
92915f71 1384
dadcd65f 1385 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
80d5c368
PM		 1386 ixgbevf_vlan_rx_add_vid(adapter->netdev,
1387 htons(ETH_P_8021Q), vid);
92915f71
GR		 1388}
1389
46ec20ff
GR		 1390static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1391{
1392 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1393 struct ixgbe_hw *hw = &adapter->hw;
1394 int count = 0;
1395
1396 if ((netdev_uc_count(netdev)) > 10) {
dbd9636e 1397 pr_err("Too many unicast filters - No Space\n");
46ec20ff
GR		 1398 return -ENOSPC;
1399 }
1400
1401 if (!netdev_uc_empty(netdev)) {
1402 struct netdev_hw_addr *ha;
1403 netdev_for_each_uc_addr(ha, netdev) {
1404 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1405 udelay(200);
1406 }
1407 } else {
1408 /*
1409 * If the list is empty then send message to PF driver to
1410 * clear all macvlans on this VF.
1411 */
1412 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1413 }
1414
1415 return count;
1416}
1417
92915f71 1418/**
dee847f5 1419 * ixgbevf_set_rx_mode - Multicast and unicast set
92915f71
GR		 1420 * @netdev: network interface device structure
1421 *
1422 * The set_rx_method entry point is called whenever the multicast address
dee847f5
GR		 1423 * list, unicast address list or the network interface flags are updated.
1424 * This routine is responsible for configuring the hardware for proper
1425 * multicast mode and configuring requested unicast filters.
92915f71
GR		 1426 **/
1427static void ixgbevf_set_rx_mode(struct net_device *netdev)
1428{
1429 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1430 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1431
55fdd45b 1432 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1433
92915f71 1434 /* reprogram multicast list */
92fe0bf7 1435 hw->mac.ops.update_mc_addr_list(hw, netdev);
46ec20ff
GR		 1436
1437 ixgbevf_write_uc_addr_list(netdev);
1c55ed76 1438
55fdd45b 1439 spin_unlock_bh(&adapter->mbx_lock);
92915f71
GR		 1440}
1441
1442static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1443{
1444 int q_idx;
1445 struct ixgbevf_q_vector *q_vector;
1446 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1447
1448 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
92915f71 1449 q_vector = adapter->q_vector[q_idx];
c777cdfa
JK		 1450#ifdef CONFIG_NET_RX_BUSY_POLL
1451 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]);
1452#endif
fa71ae27 1453 napi_enable(&q_vector->napi);
92915f71
GR		 1454 }
1455}
1456
1457static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1458{
1459 int q_idx;
1460 struct ixgbevf_q_vector *q_vector;
1461 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1462
1463 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1464 q_vector = adapter->q_vector[q_idx];
92915f71 1465 napi_disable(&q_vector->napi);
c777cdfa
JK		 1466#ifdef CONFIG_NET_RX_BUSY_POLL
1467 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) {
1468 pr_info("QV %d locked\n", q_idx);
1469 usleep_range(1000, 20000);
1470 }
1471#endif /* CONFIG_NET_RX_BUSY_POLL */
92915f71
GR		 1472 }
1473}
1474
220fe050
DS		 1475static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
1476{
1477 struct ixgbe_hw *hw = &adapter->hw;
1478 unsigned int def_q = 0;
1479 unsigned int num_tcs = 0;
1480 unsigned int num_rx_queues = 1;
1481 int err;
1482
1483 spin_lock_bh(&adapter->mbx_lock);
1484
1485 /* fetch queue configuration from the PF */
1486 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1487
1488 spin_unlock_bh(&adapter->mbx_lock);
1489
1490 if (err)
1491 return err;
1492
1493 if (num_tcs > 1) {
1494 /* update default Tx ring register index */
87e70ab9 1495 adapter->tx_ring[0]->reg_idx = def_q;
220fe050
DS		 1496
1497 /* we need as many queues as traffic classes */
1498 num_rx_queues = num_tcs;
1499 }
1500
1501 /* if we have a bad config abort request queue reset */
1502 if (adapter->num_rx_queues != num_rx_queues) {
1503 /* force mailbox timeout to prevent further messages */
1504 hw->mbx.timeout = 0;
1505
1506 /* wait for watchdog to come around and bail us out */
1507 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
1508 }
1509
1510 return 0;
1511}
1512
92915f71
GR		 1513static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1514{
220fe050
DS		 1515 ixgbevf_configure_dcb(adapter);
1516
de02decb 1517 ixgbevf_set_rx_mode(adapter->netdev);
92915f71
GR		 1518
1519 ixgbevf_restore_vlan(adapter);
1520
1521 ixgbevf_configure_tx(adapter);
1522 ixgbevf_configure_rx(adapter);
92915f71
GR		 1523}
1524
33bd9f60
GR		 1525static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1526{
1527 /* Only save pre-reset stats if there are some */
1528 if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1529 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1530 adapter->stats.base_vfgprc;
1531 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1532 adapter->stats.base_vfgptc;
1533 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1534 adapter->stats.base_vfgorc;
1535 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1536 adapter->stats.base_vfgotc;
1537 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1538 adapter->stats.base_vfmprc;
1539 }
1540}
1541
1542static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1543{
1544 struct ixgbe_hw *hw = &adapter->hw;
1545
1546 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1547 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1548 adapter->stats.last_vfgorc |=
1549 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1550 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1551 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1552 adapter->stats.last_vfgotc |=
1553 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1554 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1555
1556 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1557 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1558 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1559 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1560 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1561}
1562
31186785
AD		 1563static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1564{
1565 struct ixgbe_hw *hw = &adapter->hw;
56e94095
AD		 1566 int api[] = { ixgbe_mbox_api_11,
1567 ixgbe_mbox_api_10,
31186785
AD		 1568 ixgbe_mbox_api_unknown };
1569 int err = 0, idx = 0;
1570
55fdd45b 1571 spin_lock_bh(&adapter->mbx_lock);
31186785
AD		 1572
1573 while (api[idx] != ixgbe_mbox_api_unknown) {
1574 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1575 if (!err)
1576 break;
1577 idx++;
1578 }
1579
55fdd45b 1580 spin_unlock_bh(&adapter->mbx_lock);
31186785
AD		 1581}
1582
795180d8 1583static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
92915f71
GR		 1584{
1585 struct net_device *netdev = adapter->netdev;
1586 struct ixgbe_hw *hw = &adapter->hw;
92915f71
GR		 1587
1588 ixgbevf_configure_msix(adapter);
1589
55fdd45b 1590 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1591
92fe0bf7
GR		 1592 if (is_valid_ether_addr(hw->mac.addr))
1593 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1594 else
1595 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
92915f71 1596
55fdd45b 1597 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1598
92915f71
GR		 1599 clear_bit(__IXGBEVF_DOWN, &adapter->state);
1600 ixgbevf_napi_enable_all(adapter);
1601
1602 /* enable transmits */
1603 netif_tx_start_all_queues(netdev);
1604
33bd9f60
GR		 1605 ixgbevf_save_reset_stats(adapter);
1606 ixgbevf_init_last_counter_stats(adapter);
1607
4b2cd27f 1608 hw->mac.get_link_status = 1;
92915f71 1609 mod_timer(&adapter->watchdog_timer, jiffies);
92915f71
GR		 1610}
1611
795180d8 1612void ixgbevf_up(struct ixgbevf_adapter *adapter)
92915f71 1613{
92915f71
GR		 1614 struct ixgbe_hw *hw = &adapter->hw;
1615
1616 ixgbevf_configure(adapter);
1617
795180d8 1618 ixgbevf_up_complete(adapter);
92915f71
GR		 1619
1620 /* clear any pending interrupts, may auto mask */
1621 IXGBE_READ_REG(hw, IXGBE_VTEICR);
1622
5f3600eb 1623 ixgbevf_irq_enable(adapter);
92915f71
GR		 1624}
1625
1626/**
1627 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
92915f71
GR		 1628 * @rx_ring: ring to free buffers from
1629 **/
05d063aa 1630static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
92915f71 1631{
92915f71
GR		 1632 unsigned long size;
1633 unsigned int i;
1634
c0456c23
GR		 1635 if (!rx_ring->rx_buffer_info)
1636 return;
92915f71 1637
c0456c23 1638 /* Free all the Rx ring sk_buffs */
92915f71
GR		 1639 for (i = 0; i < rx_ring->count; i++) {
1640 struct ixgbevf_rx_buffer *rx_buffer_info;
1641
1642 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1643 if (rx_buffer_info->dma) {
05d063aa 1644 dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
92915f71 1645 rx_ring->rx_buf_len,
2a1f8794 1646 DMA_FROM_DEVICE);
92915f71
GR		 1647 rx_buffer_info->dma = 0;
1648 }
1649 if (rx_buffer_info->skb) {
1650 struct sk_buff *skb = rx_buffer_info->skb;
1651 rx_buffer_info->skb = NULL;
1652 do {
1653 struct sk_buff *this = skb;
5c60f81a 1654 skb = IXGBE_CB(skb)->prev;
92915f71
GR		 1655 dev_kfree_skb(this);
1656 } while (skb);
1657 }
92915f71
GR		 1658 }
1659
1660 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1661 memset(rx_ring->rx_buffer_info, 0, size);
1662
1663 /* Zero out the descriptor ring */
1664 memset(rx_ring->desc, 0, rx_ring->size);
92915f71
GR		 1665}
1666
1667/**
1668 * ixgbevf_clean_tx_ring - Free Tx Buffers
92915f71
GR		 1669 * @tx_ring: ring to be cleaned
1670 **/
05d063aa 1671static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
92915f71
GR		 1672{
1673 struct ixgbevf_tx_buffer *tx_buffer_info;
1674 unsigned long size;
1675 unsigned int i;
1676
c0456c23
GR		 1677 if (!tx_ring->tx_buffer_info)
1678 return;
1679
92915f71 1680 /* Free all the Tx ring sk_buffs */
92915f71
GR		 1681 for (i = 0; i < tx_ring->count; i++) {
1682 tx_buffer_info = &tx_ring->tx_buffer_info[i];
70a10e25 1683 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
92915f71
GR		 1684 }
1685
1686 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1687 memset(tx_ring->tx_buffer_info, 0, size);
1688
1689 memset(tx_ring->desc, 0, tx_ring->size);
92915f71
GR		 1690}
1691
1692/**
1693 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1694 * @adapter: board private structure
1695 **/
1696static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1697{
1698 int i;
1699
1700 for (i = 0; i < adapter->num_rx_queues; i++)
05d063aa 1701 ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
92915f71
GR		 1702}
1703
1704/**
1705 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1706 * @adapter: board private structure
1707 **/
1708static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1709{
1710 int i;
1711
1712 for (i = 0; i < adapter->num_tx_queues; i++)
05d063aa 1713 ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
92915f71
GR		 1714}
1715
1716void ixgbevf_down(struct ixgbevf_adapter *adapter)
1717{
1718 struct net_device *netdev = adapter->netdev;
1719 struct ixgbe_hw *hw = &adapter->hw;
de02decb 1720 int i;
92915f71
GR		 1721
1722 /* signal that we are down to the interrupt handler */
1723 set_bit(__IXGBEVF_DOWN, &adapter->state);
858c3dda
DS		 1724
1725 /* disable all enabled rx queues */
1726 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 1727 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]);
92915f71
GR		 1728
1729 netif_tx_disable(netdev);
1730
1731 msleep(10);
1732
1733 netif_tx_stop_all_queues(netdev);
1734
1735 ixgbevf_irq_disable(adapter);
1736
1737 ixgbevf_napi_disable_all(adapter);
1738
1739 del_timer_sync(&adapter->watchdog_timer);
1740 /* can't call flush scheduled work here because it can deadlock
1741 * if linkwatch_event tries to acquire the rtnl_lock which we are
1742 * holding */
1743 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1744 msleep(1);
1745
1746 /* disable transmits in the hardware now that interrupts are off */
1747 for (i = 0; i < adapter->num_tx_queues; i++) {
de02decb
DS		 1748 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
1749
1750 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
1751 IXGBE_TXDCTL_SWFLSH);
92915f71
GR		 1752 }
1753
1754 netif_carrier_off(netdev);
1755
1756 if (!pci_channel_offline(adapter->pdev))
1757 ixgbevf_reset(adapter);
1758
1759 ixgbevf_clean_all_tx_rings(adapter);
1760 ixgbevf_clean_all_rx_rings(adapter);
1761}
1762
1763void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1764{
1765 WARN_ON(in_interrupt());
c0456c23 1766
92915f71
GR		 1767 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1768 msleep(1);
1769
4b2cd27f
AD		 1770 ixgbevf_down(adapter);
1771 ixgbevf_up(adapter);
92915f71
GR		 1772
1773 clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1774}
1775
1776void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1777{
1778 struct ixgbe_hw *hw = &adapter->hw;
1779 struct net_device *netdev = adapter->netdev;
1780
798e381a 1781 if (hw->mac.ops.reset_hw(hw)) {
92915f71 1782 hw_dbg(hw, "PF still resetting\n");
798e381a 1783 } else {
92915f71 1784 hw->mac.ops.init_hw(hw);
798e381a
DS		 1785 ixgbevf_negotiate_api(adapter);
1786 }
92915f71
GR		 1787
1788 if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1789 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1790 netdev->addr_len);
1791 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1792 netdev->addr_len);
1793 }
1794}
1795
e45dd5fe
JK		 1796static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1797 int vectors)
92915f71 1798{
a5f9337b
ET		 1799 int err = 0;
1800 int vector_threshold;
92915f71 1801
fa71ae27
AD		 1802 /* We'll want at least 2 (vector_threshold):
1803 * 1) TxQ[0] + RxQ[0] handler
1804 * 2) Other (Link Status Change, etc.)
92915f71
GR		 1805 */
1806 vector_threshold = MIN_MSIX_COUNT;
1807
1808 /* The more we get, the more we will assign to Tx/Rx Cleanup
1809 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1810 * Right now, we simply care about how many we'll get; we'll
1811 * set them up later while requesting irq's.
1812 */
1813 while (vectors >= vector_threshold) {
1814 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1815 vectors);
e45dd5fe 1816 if (!err || err < 0) /* Success or a nasty failure. */
92915f71 1817 break;
92915f71
GR		 1818 else /* err == number of vectors we should try again with */
1819 vectors = err;
1820 }
1821
e45dd5fe
JK		 1822 if (vectors < vector_threshold)
1823 err = -ENOMEM;
1824
1825 if (err) {
1826 dev_err(&adapter->pdev->dev,
1827 "Unable to allocate MSI-X interrupts\n");
92915f71
GR		 1828 kfree(adapter->msix_entries);
1829 adapter->msix_entries = NULL;
1830 } else {
1831 /*
1832 * Adjust for only the vectors we'll use, which is minimum
1833 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1834 * vectors we were allocated.
1835 */
1836 adapter->num_msix_vectors = vectors;
1837 }
dee847f5 1838
e45dd5fe 1839 return err;
92915f71
GR		 1840}
1841
49ce9c2c
BH		 1842/**
1843 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
92915f71
GR		 1844 * @adapter: board private structure to initialize
1845 *
1846 * This is the top level queue allocation routine. The order here is very
1847 * important, starting with the "most" number of features turned on at once,
1848 * and ending with the smallest set of features. This way large combinations
1849 * can be allocated if they're turned on, and smaller combinations are the
1850 * fallthrough conditions.
1851 *
1852 **/
1853static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1854{
220fe050
DS		 1855 struct ixgbe_hw *hw = &adapter->hw;
1856 unsigned int def_q = 0;
1857 unsigned int num_tcs = 0;
1858 int err;
1859
92915f71
GR		 1860 /* Start with base case */
1861 adapter->num_rx_queues = 1;
1862 adapter->num_tx_queues = 1;
220fe050
DS		 1863
1864 spin_lock_bh(&adapter->mbx_lock);
1865
1866 /* fetch queue configuration from the PF */
1867 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1868
1869 spin_unlock_bh(&adapter->mbx_lock);
1870
1871 if (err)
1872 return;
1873
1874 /* we need as many queues as traffic classes */
1875 if (num_tcs > 1)
1876 adapter->num_rx_queues = num_tcs;
92915f71
GR		 1877}
1878
1879/**
1880 * ixgbevf_alloc_queues - Allocate memory for all rings
1881 * @adapter: board private structure to initialize
1882 *
1883 * We allocate one ring per queue at run-time since we don't know the
1884 * number of queues at compile-time. The polling_netdev array is
1885 * intended for Multiqueue, but should work fine with a single queue.
1886 **/
1887static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1888{
87e70ab9
DS		 1889 struct ixgbevf_ring *ring;
1890 int rx = 0, tx = 0;
92915f71 1891
87e70ab9
DS		 1892 for (; tx < adapter->num_tx_queues; tx++) {
1893 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1894 if (!ring)
1895 goto err_allocation;
92915f71 1896
87e70ab9
DS		 1897 ring->dev = &adapter->pdev->dev;
1898 ring->netdev = adapter->netdev;
1899 ring->count = adapter->tx_ring_count;
1900 ring->queue_index = tx;
1901 ring->reg_idx = tx;
92915f71 1902
87e70ab9 1903 adapter->tx_ring[tx] = ring;
92915f71
GR		 1904 }
1905
87e70ab9
DS		 1906 for (; rx < adapter->num_rx_queues; rx++) {
1907 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1908 if (!ring)
1909 goto err_allocation;
1910
1911 ring->dev = &adapter->pdev->dev;
1912 ring->netdev = adapter->netdev;
1913
1914 ring->count = adapter->rx_ring_count;
1915 ring->queue_index = rx;
1916 ring->reg_idx = rx;
1917
1918 adapter->rx_ring[rx] = ring;
92915f71
GR		 1919 }
1920
1921 return 0;
1922
87e70ab9
DS		 1923err_allocation:
1924 while (tx) {
1925 kfree(adapter->tx_ring[--tx]);
1926 adapter->tx_ring[tx] = NULL;
1927 }
1928
1929 while (rx) {
1930 kfree(adapter->rx_ring[--rx]);
1931 adapter->rx_ring[rx] = NULL;
1932 }
92915f71
GR		 1933 return -ENOMEM;
1934}
1935
1936/**
1937 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1938 * @adapter: board private structure to initialize
1939 *
1940 * Attempt to configure the interrupts using the best available
1941 * capabilities of the hardware and the kernel.
1942 **/
1943static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1944{
91e2b89b 1945 struct net_device *netdev = adapter->netdev;
92915f71
GR		 1946 int err = 0;
1947 int vector, v_budget;
1948
1949 /*
1950 * It's easy to be greedy for MSI-X vectors, but it really
1951 * doesn't do us much good if we have a lot more vectors
1952 * than CPU's. So let's be conservative and only ask for
fa71ae27
AD		 1953 * (roughly) the same number of vectors as there are CPU's.
1954 * The default is to use pairs of vectors.
92915f71 1955 */
fa71ae27
AD		 1956 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1957 v_budget = min_t(int, v_budget, num_online_cpus());
1958 v_budget += NON_Q_VECTORS;
92915f71
GR		 1959
1960 /* A failure in MSI-X entry allocation isn't fatal, but it does
1961 * mean we disable MSI-X capabilities of the adapter. */
1962 adapter->msix_entries = kcalloc(v_budget,
1963 sizeof(struct msix_entry), GFP_KERNEL);
1964 if (!adapter->msix_entries) {
1965 err = -ENOMEM;
1966 goto out;
1967 }
1968
1969 for (vector = 0; vector < v_budget; vector++)
1970 adapter->msix_entries[vector].entry = vector;
1971
e45dd5fe
JK		 1972 err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
1973 if (err)
1974 goto out;
92915f71 1975
91e2b89b
GR		 1976 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
1977 if (err)
1978 goto out;
1979
1980 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
1981
92915f71
GR		 1982out:
1983 return err;
1984}
1985
1986/**
1987 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
1988 * @adapter: board private structure to initialize
1989 *
1990 * We allocate one q_vector per queue interrupt. If allocation fails we
1991 * return -ENOMEM.
1992 **/
1993static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
1994{
1995 int q_idx, num_q_vectors;
1996 struct ixgbevf_q_vector *q_vector;
92915f71
GR		 1997
1998 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 1999
2000 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2001 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2002 if (!q_vector)
2003 goto err_out;
2004 q_vector->adapter = adapter;
2005 q_vector->v_idx = q_idx;
fa71ae27
AD		 2006 netif_napi_add(adapter->netdev, &q_vector->napi,
2007 ixgbevf_poll, 64);
c777cdfa
JK		 2008#ifdef CONFIG_NET_RX_BUSY_POLL
2009 napi_hash_add(&q_vector->napi);
2010#endif
92915f71
GR		 2011 adapter->q_vector[q_idx] = q_vector;
2012 }
2013
2014 return 0;
2015
2016err_out:
2017 while (q_idx) {
2018 q_idx--;
2019 q_vector = adapter->q_vector[q_idx];
c777cdfa
JK		 2020#ifdef CONFIG_NET_RX_BUSY_POLL
2021 napi_hash_del(&q_vector->napi);
2022#endif
92915f71
GR		 2023 netif_napi_del(&q_vector->napi);
2024 kfree(q_vector);
2025 adapter->q_vector[q_idx] = NULL;
2026 }
2027 return -ENOMEM;
2028}
2029
2030/**
2031 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2032 * @adapter: board private structure to initialize
2033 *
2034 * This function frees the memory allocated to the q_vectors. In addition if
2035 * NAPI is enabled it will delete any references to the NAPI struct prior
2036 * to freeing the q_vector.
2037 **/
2038static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2039{
f4477702 2040 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 2041
2042 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2043 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2044
2045 adapter->q_vector[q_idx] = NULL;
c777cdfa
JK		 2046#ifdef CONFIG_NET_RX_BUSY_POLL
2047 napi_hash_del(&q_vector->napi);
2048#endif
f4477702 2049 netif_napi_del(&q_vector->napi);
92915f71
GR		 2050 kfree(q_vector);
2051 }
2052}
2053
2054/**
2055 * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2056 * @adapter: board private structure
2057 *
2058 **/
2059static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2060{
2061 pci_disable_msix(adapter->pdev);
2062 kfree(adapter->msix_entries);
2063 adapter->msix_entries = NULL;
92915f71
GR		 2064}
2065
2066/**
2067 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2068 * @adapter: board private structure to initialize
2069 *
2070 **/
2071static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2072{
2073 int err;
2074
2075 /* Number of supported queues */
2076 ixgbevf_set_num_queues(adapter);
2077
2078 err = ixgbevf_set_interrupt_capability(adapter);
2079 if (err) {
2080 hw_dbg(&adapter->hw,
2081 "Unable to setup interrupt capabilities\n");
2082 goto err_set_interrupt;
2083 }
2084
2085 err = ixgbevf_alloc_q_vectors(adapter);
2086 if (err) {
2087 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2088 "vectors\n");
2089 goto err_alloc_q_vectors;
2090 }
2091
2092 err = ixgbevf_alloc_queues(adapter);
2093 if (err) {
dbd9636e 2094 pr_err("Unable to allocate memory for queues\n");
92915f71
GR		 2095 goto err_alloc_queues;
2096 }
2097
2098 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2099 "Tx Queue count = %u\n",
2100 (adapter->num_rx_queues > 1) ? "Enabled" :
2101 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2102
2103 set_bit(__IXGBEVF_DOWN, &adapter->state);
2104
2105 return 0;
2106err_alloc_queues:
2107 ixgbevf_free_q_vectors(adapter);
2108err_alloc_q_vectors:
2109 ixgbevf_reset_interrupt_capability(adapter);
2110err_set_interrupt:
2111 return err;
2112}
2113
0ac1e8ce
AD		 2114/**
2115 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2116 * @adapter: board private structure to clear interrupt scheme on
2117 *
2118 * We go through and clear interrupt specific resources and reset the structure
2119 * to pre-load conditions
2120 **/
2121static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2122{
87e70ab9
DS		 2123 int i;
2124
2125 for (i = 0; i < adapter->num_tx_queues; i++) {
2126 kfree(adapter->tx_ring[i]);
2127 adapter->tx_ring[i] = NULL;
2128 }
2129 for (i = 0; i < adapter->num_rx_queues; i++) {
2130 kfree(adapter->rx_ring[i]);
2131 adapter->rx_ring[i] = NULL;
2132 }
2133
0ac1e8ce
AD		 2134 adapter->num_tx_queues = 0;
2135 adapter->num_rx_queues = 0;
2136
2137 ixgbevf_free_q_vectors(adapter);
2138 ixgbevf_reset_interrupt_capability(adapter);
2139}
2140
92915f71
GR		 2141/**
2142 * ixgbevf_sw_init - Initialize general software structures
2143 * (struct ixgbevf_adapter)
2144 * @adapter: board private structure to initialize
2145 *
2146 * ixgbevf_sw_init initializes the Adapter private data structure.
2147 * Fields are initialized based on PCI device information and
2148 * OS network device settings (MTU size).
2149 **/
9f9a12f8 2150static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
92915f71
GR		 2151{
2152 struct ixgbe_hw *hw = &adapter->hw;
2153 struct pci_dev *pdev = adapter->pdev;
e1941a74 2154 struct net_device *netdev = adapter->netdev;
92915f71
GR		 2155 int err;
2156
2157 /* PCI config space info */
2158
2159 hw->vendor_id = pdev->vendor;
2160 hw->device_id = pdev->device;
ff938e43 2161 hw->revision_id = pdev->revision;
92915f71
GR		 2162 hw->subsystem_vendor_id = pdev->subsystem_vendor;
2163 hw->subsystem_device_id = pdev->subsystem_device;
2164
2165 hw->mbx.ops.init_params(hw);
56e94095
AD		 2166
2167 /* assume legacy case in which PF would only give VF 2 queues */
2168 hw->mac.max_tx_queues = 2;
2169 hw->mac.max_rx_queues = 2;
2170
798e381a
DS		 2171 /* lock to protect mailbox accesses */
2172 spin_lock_init(&adapter->mbx_lock);
2173
92915f71
GR		 2174 err = hw->mac.ops.reset_hw(hw);
2175 if (err) {
2176 dev_info(&pdev->dev,
e1941a74 2177 "PF still in reset state. Is the PF interface up?\n");
92915f71
GR		 2178 } else {
2179 err = hw->mac.ops.init_hw(hw);
2180 if (err) {
dbd9636e 2181 pr_err("init_shared_code failed: %d\n", err);
92915f71
GR		 2182 goto out;
2183 }
798e381a 2184 ixgbevf_negotiate_api(adapter);
e1941a74
GR		 2185 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
2186 if (err)
2187 dev_info(&pdev->dev, "Error reading MAC address\n");
2188 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2189 dev_info(&pdev->dev,
2190 "MAC address not assigned by administrator.\n");
2191 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2192 }
2193
2194 if (!is_valid_ether_addr(netdev->dev_addr)) {
2195 dev_info(&pdev->dev, "Assigning random MAC address\n");
2196 eth_hw_addr_random(netdev);
2197 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len);
92915f71
GR		 2198 }
2199
2200 /* Enable dynamic interrupt throttling rates */
5f3600eb
AD		 2201 adapter->rx_itr_setting = 1;
2202 adapter->tx_itr_setting = 1;
92915f71 2203
92915f71
GR		 2204 /* set default ring sizes */
2205 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2206 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2207
92915f71 2208 set_bit(__IXGBEVF_DOWN, &adapter->state);
1a0d6ae5 2209 return 0;
92915f71
GR		 2210
2211out:
2212 return err;
2213}
2214
92915f71
GR		 2215#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
2216 { \
2217 u32 current_counter = IXGBE_READ_REG(hw, reg); \
2218 if (current_counter < last_counter) \
2219 counter += 0x100000000LL; \
2220 last_counter = current_counter; \
2221 counter &= 0xFFFFFFFF00000000LL; \
2222 counter |= current_counter; \
2223 }
2224
2225#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2226 { \
2227 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \
2228 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \
2229 u64 current_counter = (current_counter_msb << 32) | \
2230 current_counter_lsb; \
2231 if (current_counter < last_counter) \
2232 counter += 0x1000000000LL; \
2233 last_counter = current_counter; \
2234 counter &= 0xFFFFFFF000000000LL; \
2235 counter |= current_counter; \
2236 }
2237/**
2238 * ixgbevf_update_stats - Update the board statistics counters.
2239 * @adapter: board private structure
2240 **/
2241void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2242{
2243 struct ixgbe_hw *hw = &adapter->hw;
55fb277c 2244 int i;
92915f71 2245
088245a3
GR		 2246 if (!adapter->link_up)
2247 return;
2248
92915f71
GR		 2249 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2250 adapter->stats.vfgprc);
2251 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2252 adapter->stats.vfgptc);
2253 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2254 adapter->stats.last_vfgorc,
2255 adapter->stats.vfgorc);
2256 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2257 adapter->stats.last_vfgotc,
2258 adapter->stats.vfgotc);
2259 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2260 adapter->stats.vfmprc);
55fb277c
GR		 2261
2262 for (i = 0; i < adapter->num_rx_queues; i++) {
2263 adapter->hw_csum_rx_error +=
87e70ab9 2264 adapter->rx_ring[i]->hw_csum_rx_error;
87e70ab9 2265 adapter->rx_ring[i]->hw_csum_rx_error = 0;
55fb277c 2266 }
92915f71
GR		 2267}
2268
2269/**
2270 * ixgbevf_watchdog - Timer Call-back
2271 * @data: pointer to adapter cast into an unsigned long
2272 **/
2273static void ixgbevf_watchdog(unsigned long data)
2274{
2275 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2276 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 2277 u32 eics = 0;
92915f71
GR		 2278 int i;
2279
2280 /*
2281 * Do the watchdog outside of interrupt context due to the lovely
2282 * delays that some of the newer hardware requires
2283 */
2284
2285 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2286 goto watchdog_short_circuit;
2287
2288 /* get one bit for every active tx/rx interrupt vector */
2289 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2290 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
6b43c446 2291 if (qv->rx.ring || qv->tx.ring)
5f3600eb 2292 eics |= 1 << i;
92915f71
GR		 2293 }
2294
5f3600eb 2295 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
92915f71
GR		 2296
2297watchdog_short_circuit:
2298 schedule_work(&adapter->watchdog_task);
2299}
2300
2301/**
2302 * ixgbevf_tx_timeout - Respond to a Tx Hang
2303 * @netdev: network interface device structure
2304 **/
2305static void ixgbevf_tx_timeout(struct net_device *netdev)
2306{
2307 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2308
2309 /* Do the reset outside of interrupt context */
2310 schedule_work(&adapter->reset_task);
2311}
2312
2313static void ixgbevf_reset_task(struct work_struct *work)
2314{
2315 struct ixgbevf_adapter *adapter;
2316 adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2317
2318 /* If we're already down or resetting, just bail */
2319 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2320 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2321 return;
2322
2323 adapter->tx_timeout_count++;
2324
2325 ixgbevf_reinit_locked(adapter);
2326}
2327
2328/**
2329 * ixgbevf_watchdog_task - worker thread to bring link up
2330 * @work: pointer to work_struct containing our data
2331 **/
2332static void ixgbevf_watchdog_task(struct work_struct *work)
2333{
2334 struct ixgbevf_adapter *adapter = container_of(work,
2335 struct ixgbevf_adapter,
2336 watchdog_task);
2337 struct net_device *netdev = adapter->netdev;
2338 struct ixgbe_hw *hw = &adapter->hw;
2339 u32 link_speed = adapter->link_speed;
2340 bool link_up = adapter->link_up;
92fe0bf7 2341 s32 need_reset;
92915f71 2342
220fe050
DS		 2343 ixgbevf_queue_reset_subtask(adapter);
2344
92915f71
GR		 2345 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2346
2347 /*
2348 * Always check the link on the watchdog because we have
2349 * no LSC interrupt
2350 */
92fe0bf7 2351 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 2352
92fe0bf7 2353 need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
1c55ed76 2354
92fe0bf7 2355 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 2356
92fe0bf7
GR		 2357 if (need_reset) {
2358 adapter->link_up = link_up;
2359 adapter->link_speed = link_speed;
2360 netif_carrier_off(netdev);
2361 netif_tx_stop_all_queues(netdev);
2362 schedule_work(&adapter->reset_task);
2363 goto pf_has_reset;
92915f71
GR		 2364 }
2365 adapter->link_up = link_up;
2366 adapter->link_speed = link_speed;
2367
2368 if (link_up) {
2369 if (!netif_carrier_ok(netdev)) {
b876a744
GR		 2370 char *link_speed_string;
2371 switch (link_speed) {
2372 case IXGBE_LINK_SPEED_10GB_FULL:
2373 link_speed_string = "10 Gbps";
2374 break;
2375 case IXGBE_LINK_SPEED_1GB_FULL:
2376 link_speed_string = "1 Gbps";
2377 break;
2378 case IXGBE_LINK_SPEED_100_FULL:
2379 link_speed_string = "100 Mbps";
2380 break;
2381 default:
2382 link_speed_string = "unknown speed";
2383 break;
2384 }
6fe59675 2385 dev_info(&adapter->pdev->dev,
b876a744 2386 "NIC Link is Up, %s\n", link_speed_string);
92915f71
GR		 2387 netif_carrier_on(netdev);
2388 netif_tx_wake_all_queues(netdev);
92915f71
GR		 2389 }
2390 } else {
2391 adapter->link_up = false;
2392 adapter->link_speed = 0;
2393 if (netif_carrier_ok(netdev)) {
6fe59675 2394 dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
92915f71
GR		 2395 netif_carrier_off(netdev);
2396 netif_tx_stop_all_queues(netdev);
2397 }
2398 }
2399
92915f71
GR		 2400 ixgbevf_update_stats(adapter);
2401
33bd9f60 2402pf_has_reset:
92915f71
GR		 2403 /* Reset the timer */
2404 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2405 mod_timer(&adapter->watchdog_timer,
2406 round_jiffies(jiffies + (2 * HZ)));
2407
2408 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2409}
2410
2411/**
2412 * ixgbevf_free_tx_resources - Free Tx Resources per Queue
92915f71
GR		 2413 * @tx_ring: Tx descriptor ring for a specific queue
2414 *
2415 * Free all transmit software resources
2416 **/
05d063aa 2417void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
92915f71 2418{
05d063aa 2419 ixgbevf_clean_tx_ring(tx_ring);
92915f71
GR		 2420
2421 vfree(tx_ring->tx_buffer_info);
2422 tx_ring->tx_buffer_info = NULL;
2423
de02decb
DS		 2424 /* if not set, then don't free */
2425 if (!tx_ring->desc)
2426 return;
2427
05d063aa 2428 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
2a1f8794 2429 tx_ring->dma);
92915f71
GR		 2430
2431 tx_ring->desc = NULL;
2432}
2433
2434/**
2435 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2436 * @adapter: board private structure
2437 *
2438 * Free all transmit software resources
2439 **/
2440static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2441{
2442 int i;
2443
2444 for (i = 0; i < adapter->num_tx_queues; i++)
87e70ab9 2445 if (adapter->tx_ring[i]->desc)
05d063aa 2446 ixgbevf_free_tx_resources(adapter->tx_ring[i]);
92915f71
GR		 2447}
2448
2449/**
2450 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
92915f71
GR		 2451 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2452 *
2453 * Return 0 on success, negative on failure
2454 **/
05d063aa 2455int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
92915f71 2456{
92915f71
GR		 2457 int size;
2458
2459 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
89bf67f1 2460 tx_ring->tx_buffer_info = vzalloc(size);
92915f71
GR		 2461 if (!tx_ring->tx_buffer_info)
2462 goto err;
92915f71
GR		 2463
2464 /* round up to nearest 4K */
2465 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2466 tx_ring->size = ALIGN(tx_ring->size, 4096);
2467
05d063aa 2468 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,
2a1f8794 2469 &tx_ring->dma, GFP_KERNEL);
92915f71
GR		 2470 if (!tx_ring->desc)
2471 goto err;
2472
92915f71
GR		 2473 return 0;
2474
2475err:
2476 vfree(tx_ring->tx_buffer_info);
2477 tx_ring->tx_buffer_info = NULL;
2478 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2479 "descriptor ring\n");
2480 return -ENOMEM;
2481}
2482
2483/**
2484 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2485 * @adapter: board private structure
2486 *
2487 * If this function returns with an error, then it's possible one or
2488 * more of the rings is populated (while the rest are not). It is the
2489 * callers duty to clean those orphaned rings.
2490 *
2491 * Return 0 on success, negative on failure
2492 **/
2493static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2494{
2495 int i, err = 0;
2496
2497 for (i = 0; i < adapter->num_tx_queues; i++) {
05d063aa 2498 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);
92915f71
GR		 2499 if (!err)
2500 continue;
2501 hw_dbg(&adapter->hw,
2502 "Allocation for Tx Queue %u failed\n", i);
2503 break;
2504 }
2505
2506 return err;
2507}
2508
2509/**
2510 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
92915f71
GR		 2511 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2512 *
2513 * Returns 0 on success, negative on failure
2514 **/
05d063aa 2515int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring)
92915f71 2516{
92915f71
GR		 2517 int size;
2518
2519 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
89bf67f1 2520 rx_ring->rx_buffer_info = vzalloc(size);
e404decb 2521 if (!rx_ring->rx_buffer_info)
05d063aa 2522 goto err;
92915f71
GR		 2523
2524 /* Round up to nearest 4K */
2525 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2526 rx_ring->size = ALIGN(rx_ring->size, 4096);
2527
05d063aa 2528 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size,
2a1f8794 2529 &rx_ring->dma, GFP_KERNEL);
92915f71 2530
05d063aa
ET		 2531 if (!rx_ring->desc)
2532 goto err;
92915f71 2533
92915f71 2534 return 0;
05d063aa
ET		 2535err:
2536 vfree(rx_ring->rx_buffer_info);
2537 rx_ring->rx_buffer_info = NULL;
2538 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
92915f71
GR		 2539 return -ENOMEM;
2540}
2541
2542/**
2543 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2544 * @adapter: board private structure
2545 *
2546 * If this function returns with an error, then it's possible one or
2547 * more of the rings is populated (while the rest are not). It is the
2548 * callers duty to clean those orphaned rings.
2549 *
2550 * Return 0 on success, negative on failure
2551 **/
2552static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2553{
2554 int i, err = 0;
2555
2556 for (i = 0; i < adapter->num_rx_queues; i++) {
05d063aa 2557 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]);
92915f71
GR		 2558 if (!err)
2559 continue;
2560 hw_dbg(&adapter->hw,
2561 "Allocation for Rx Queue %u failed\n", i);
2562 break;
2563 }
2564 return err;
2565}
2566
2567/**
2568 * ixgbevf_free_rx_resources - Free Rx Resources
92915f71
GR		 2569 * @rx_ring: ring to clean the resources from
2570 *
2571 * Free all receive software resources
2572 **/
05d063aa 2573void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
92915f71 2574{
05d063aa 2575 ixgbevf_clean_rx_ring(rx_ring);
92915f71
GR		 2576
2577 vfree(rx_ring->rx_buffer_info);
2578 rx_ring->rx_buffer_info = NULL;
2579
05d063aa 2580 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
2a1f8794 2581 rx_ring->dma);
92915f71
GR		 2582
2583 rx_ring->desc = NULL;
2584}
2585
2586/**
2587 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2588 * @adapter: board private structure
2589 *
2590 * Free all receive software resources
2591 **/
2592static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2593{
2594 int i;
2595
2596 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 2597 if (adapter->rx_ring[i]->desc)
05d063aa 2598 ixgbevf_free_rx_resources(adapter->rx_ring[i]);
92915f71
GR		 2599}
2600
2601/**
2602 * ixgbevf_open - Called when a network interface is made active
2603 * @netdev: network interface device structure
2604 *
2605 * Returns 0 on success, negative value on failure
2606 *
2607 * The open entry point is called when a network interface is made
2608 * active by the system (IFF_UP). At this point all resources needed
2609 * for transmit and receive operations are allocated, the interrupt
2610 * handler is registered with the OS, the watchdog timer is started,
2611 * and the stack is notified that the interface is ready.
2612 **/
2613static int ixgbevf_open(struct net_device *netdev)
2614{
2615 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2616 struct ixgbe_hw *hw = &adapter->hw;
2617 int err;
2618
a1f6c6b1 2619 /* A previous failure to open the device because of a lack of
2620 * available MSIX vector resources may have reset the number
2621 * of msix vectors variable to zero. The only way to recover
2622 * is to unload/reload the driver and hope that the system has
2623 * been able to recover some MSIX vector resources.
2624 */
2625 if (!adapter->num_msix_vectors)
2626 return -ENOMEM;
2627
92915f71
GR		 2628 /* disallow open during test */
2629 if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2630 return -EBUSY;
2631
2632 if (hw->adapter_stopped) {
2633 ixgbevf_reset(adapter);
2634 /* if adapter is still stopped then PF isn't up and
2635 * the vf can't start. */
2636 if (hw->adapter_stopped) {
2637 err = IXGBE_ERR_MBX;
dbd9636e
JK		 2638 pr_err("Unable to start - perhaps the PF Driver isn't "
2639 "up yet\n");
92915f71
GR		 2640 goto err_setup_reset;
2641 }
2642 }
2643
2644 /* allocate transmit descriptors */
2645 err = ixgbevf_setup_all_tx_resources(adapter);
2646 if (err)
2647 goto err_setup_tx;
2648
2649 /* allocate receive descriptors */
2650 err = ixgbevf_setup_all_rx_resources(adapter);
2651 if (err)
2652 goto err_setup_rx;
2653
2654 ixgbevf_configure(adapter);
2655
2656 /*
2657 * Map the Tx/Rx rings to the vectors we were allotted.
2658 * if request_irq will be called in this function map_rings
2659 * must be called *before* up_complete
2660 */
2661 ixgbevf_map_rings_to_vectors(adapter);
2662
795180d8 2663 ixgbevf_up_complete(adapter);
92915f71
GR		 2664
2665 /* clear any pending interrupts, may auto mask */
2666 IXGBE_READ_REG(hw, IXGBE_VTEICR);
2667 err = ixgbevf_request_irq(adapter);
2668 if (err)
2669 goto err_req_irq;
2670
5f3600eb 2671 ixgbevf_irq_enable(adapter);
92915f71
GR		 2672
2673 return 0;
2674
2675err_req_irq:
2676 ixgbevf_down(adapter);
92915f71
GR		 2677err_setup_rx:
2678 ixgbevf_free_all_rx_resources(adapter);
2679err_setup_tx:
2680 ixgbevf_free_all_tx_resources(adapter);
2681 ixgbevf_reset(adapter);
2682
2683err_setup_reset:
2684
2685 return err;
2686}
2687
2688/**
2689 * ixgbevf_close - Disables a network interface
2690 * @netdev: network interface device structure
2691 *
2692 * Returns 0, this is not allowed to fail
2693 *
2694 * The close entry point is called when an interface is de-activated
2695 * by the OS. The hardware is still under the drivers control, but
2696 * needs to be disabled. A global MAC reset is issued to stop the
2697 * hardware, and all transmit and receive resources are freed.
2698 **/
2699static int ixgbevf_close(struct net_device *netdev)
2700{
2701 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2702
2703 ixgbevf_down(adapter);
2704 ixgbevf_free_irq(adapter);
2705
2706 ixgbevf_free_all_tx_resources(adapter);
2707 ixgbevf_free_all_rx_resources(adapter);
2708
2709 return 0;
2710}
2711
220fe050
DS		 2712static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter)
2713{
2714 struct net_device *dev = adapter->netdev;
2715
2716 if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED))
2717 return;
2718
2719 adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
2720
2721 /* if interface is down do nothing */
2722 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2723 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2724 return;
2725
2726 /* Hardware has to reinitialize queues and interrupts to
2727 * match packet buffer alignment. Unfortunately, the
2728 * hardware is not flexible enough to do this dynamically.
2729 */
2730 if (netif_running(dev))
2731 ixgbevf_close(dev);
2732
2733 ixgbevf_clear_interrupt_scheme(adapter);
2734 ixgbevf_init_interrupt_scheme(adapter);
2735
2736 if (netif_running(dev))
2737 ixgbevf_open(dev);
2738}
2739
70a10e25
AD		 2740static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2741 u32 vlan_macip_lens, u32 type_tucmd,
2742 u32 mss_l4len_idx)
92915f71
GR		 2743{
2744 struct ixgbe_adv_tx_context_desc *context_desc;
70a10e25 2745 u16 i = tx_ring->next_to_use;
92915f71 2746
70a10e25 2747 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
92915f71 2748
70a10e25
AD		 2749 i++;
2750 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
92915f71 2751
70a10e25
AD		 2752 /* set bits to identify this as an advanced context descriptor */
2753 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
92915f71 2754
70a10e25
AD		 2755 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2756 context_desc->seqnum_seed = 0;
2757 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
2758 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2759}
2760
2761static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
2762 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2763{
2764 u32 vlan_macip_lens, type_tucmd;
2765 u32 mss_l4len_idx, l4len;
2766
2767 if (!skb_is_gso(skb))
2768 return 0;
92915f71 2769
70a10e25
AD		 2770 if (skb_header_cloned(skb)) {
2771 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2772 if (err)
2773 return err;
92915f71
GR		 2774 }
2775
70a10e25
AD		 2776 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2777 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2778
2779 if (skb->protocol == htons(ETH_P_IP)) {
2780 struct iphdr *iph = ip_hdr(skb);
2781 iph->tot_len = 0;
2782 iph->check = 0;
2783 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2784 iph->daddr, 0,
2785 IPPROTO_TCP,
2786 0);
2787 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2788 } else if (skb_is_gso_v6(skb)) {
2789 ipv6_hdr(skb)->payload_len = 0;
2790 tcp_hdr(skb)->check =
2791 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2792 &ipv6_hdr(skb)->daddr,
2793 0, IPPROTO_TCP, 0);
2794 }
2795
2796 /* compute header lengths */
2797 l4len = tcp_hdrlen(skb);
2798 *hdr_len += l4len;
2799 *hdr_len = skb_transport_offset(skb) + l4len;
2800
2801 /* mss_l4len_id: use 1 as index for TSO */
2802 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2803 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2804 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2805
2806 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2807 vlan_macip_lens = skb_network_header_len(skb);
2808 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2809 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2810
2811 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2812 type_tucmd, mss_l4len_idx);
2813
2814 return 1;
92915f71
GR		 2815}
2816
70a10e25 2817static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
92915f71
GR		 2818 struct sk_buff *skb, u32 tx_flags)
2819{
70a10e25
AD		 2820 u32 vlan_macip_lens = 0;
2821 u32 mss_l4len_idx = 0;
2822 u32 type_tucmd = 0;
92915f71 2823
70a10e25
AD		 2824 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2825 u8 l4_hdr = 0;
2826 switch (skb->protocol) {
2827 case __constant_htons(ETH_P_IP):
2828 vlan_macip_lens |= skb_network_header_len(skb);
2829 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2830 l4_hdr = ip_hdr(skb)->protocol;
2831 break;
2832 case __constant_htons(ETH_P_IPV6):
2833 vlan_macip_lens |= skb_network_header_len(skb);
2834 l4_hdr = ipv6_hdr(skb)->nexthdr;
2835 break;
2836 default:
2837 if (unlikely(net_ratelimit())) {
2838 dev_warn(tx_ring->dev,
2839 "partial checksum but proto=%x!\n",
2840 skb->protocol);
2841 }
2842 break;
2843 }
92915f71 2844
70a10e25
AD		 2845 switch (l4_hdr) {
2846 case IPPROTO_TCP:
2847 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2848 mss_l4len_idx = tcp_hdrlen(skb) <<
2849 IXGBE_ADVTXD_L4LEN_SHIFT;
2850 break;
2851 case IPPROTO_SCTP:
2852 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2853 mss_l4len_idx = sizeof(struct sctphdr) <<
2854 IXGBE_ADVTXD_L4LEN_SHIFT;
2855 break;
2856 case IPPROTO_UDP:
2857 mss_l4len_idx = sizeof(struct udphdr) <<
2858 IXGBE_ADVTXD_L4LEN_SHIFT;
2859 break;
2860 default:
2861 if (unlikely(net_ratelimit())) {
2862 dev_warn(tx_ring->dev,
2863 "partial checksum but l4 proto=%x!\n",
2864 l4_hdr);
2865 }
2866 break;
2867 }
92915f71
GR		 2868 }
2869
70a10e25
AD		 2870 /* vlan_macip_lens: MACLEN, VLAN tag */
2871 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2872 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2873
2874 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2875 type_tucmd, mss_l4len_idx);
2876
2877 return (skb->ip_summed == CHECKSUM_PARTIAL);
92915f71
GR		 2878}
2879
70a10e25 2880static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
e757e3e1 2881 struct sk_buff *skb, u32 tx_flags)
92915f71 2882{
92915f71
GR		 2883 struct ixgbevf_tx_buffer *tx_buffer_info;
2884 unsigned int len;
2885 unsigned int total = skb->len;
2540ddb5
KV		 2886 unsigned int offset = 0, size;
2887 int count = 0;
92915f71
GR		 2888 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2889 unsigned int f;
65deeed7 2890 int i;
92915f71
GR		 2891
2892 i = tx_ring->next_to_use;
2893
2894 len = min(skb_headlen(skb), total);
2895 while (len) {
2896 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2897 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2898
2899 tx_buffer_info->length = size;
2900 tx_buffer_info->mapped_as_page = false;
70a10e25 2901 tx_buffer_info->dma = dma_map_single(tx_ring->dev,
92915f71 2902 skb->data + offset,
2a1f8794 2903 size, DMA_TO_DEVICE);
70a10e25 2904 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma))
92915f71 2905 goto dma_error;
92915f71
GR		 2906
2907 len -= size;
2908 total -= size;
2909 offset += size;
2910 count++;
2911 i++;
2912 if (i == tx_ring->count)
2913 i = 0;
2914 }
2915
2916 for (f = 0; f < nr_frags; f++) {
9e903e08 2917 const struct skb_frag_struct *frag;
92915f71
GR		 2918
2919 frag = &skb_shinfo(skb)->frags[f];
9e903e08 2920 len = min((unsigned int)skb_frag_size(frag), total);
877749bf 2921 offset = 0;
92915f71
GR		 2922
2923 while (len) {
2924 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2925 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2926
2927 tx_buffer_info->length = size;
877749bf 2928 tx_buffer_info->dma =
70a10e25 2929 skb_frag_dma_map(tx_ring->dev, frag,
877749bf 2930 offset, size, DMA_TO_DEVICE);
70a10e25
AD		 2931 if (dma_mapping_error(tx_ring->dev,
2932 tx_buffer_info->dma))
92915f71 2933 goto dma_error;
6132ee8a 2934 tx_buffer_info->mapped_as_page = true;
92915f71
GR		 2935
2936 len -= size;
2937 total -= size;
2938 offset += size;
2939 count++;
2940 i++;
2941 if (i == tx_ring->count)
2942 i = 0;
2943 }
2944 if (total == 0)
2945 break;
2946 }
2947
2948 if (i == 0)
2949 i = tx_ring->count - 1;
2950 else
2951 i = i - 1;
2952 tx_ring->tx_buffer_info[i].skb = skb;
92915f71
GR		 2953
2954 return count;
2955
2956dma_error:
70a10e25 2957 dev_err(tx_ring->dev, "TX DMA map failed\n");
92915f71
GR		 2958
2959 /* clear timestamp and dma mappings for failed tx_buffer_info map */
2960 tx_buffer_info->dma = 0;
92915f71
GR		 2961 count--;
2962
2963 /* clear timestamp and dma mappings for remaining portion of packet */
2964 while (count >= 0) {
2965 count--;
2966 i--;
2967 if (i < 0)
2968 i += tx_ring->count;
2969 tx_buffer_info = &tx_ring->tx_buffer_info[i];
70a10e25 2970 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
92915f71
GR		 2971 }
2972
2973 return count;
2974}
2975
70a10e25 2976static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags,
e757e3e1
AD		 2977 int count, unsigned int first, u32 paylen,
2978 u8 hdr_len)
92915f71
GR		 2979{
2980 union ixgbe_adv_tx_desc *tx_desc = NULL;
2981 struct ixgbevf_tx_buffer *tx_buffer_info;
2982 u32 olinfo_status = 0, cmd_type_len = 0;
2983 unsigned int i;
2984
2985 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
2986
2987 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
2988
2989 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
2990
2991 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2992 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
2993
70a10e25
AD		 2994 if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2995 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM;
2996
92915f71
GR		 2997 if (tx_flags & IXGBE_TX_FLAGS_TSO) {
2998 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
2999
92915f71
GR		 3000 /* use index 1 context for tso */
3001 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
3002 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
70a10e25 3003 olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM;
70a10e25 3004 }
92915f71 3005
70a10e25
AD		 3006 /*
3007 * Check Context must be set if Tx switch is enabled, which it
3008 * always is for case where virtual functions are running
3009 */
3010 olinfo_status |= IXGBE_ADVTXD_CC;
92915f71
GR		 3011
3012 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
3013
3014 i = tx_ring->next_to_use;
3015 while (count--) {
3016 tx_buffer_info = &tx_ring->tx_buffer_info[i];
908421f6 3017 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
92915f71
GR		 3018 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
3019 tx_desc->read.cmd_type_len =
3020 cpu_to_le32(cmd_type_len | tx_buffer_info->length);
3021 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3022 i++;
3023 if (i == tx_ring->count)
3024 i = 0;
3025 }
3026
3027 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
3028
e757e3e1
AD		 3029 tx_ring->tx_buffer_info[first].time_stamp = jiffies;
3030
3031 /* Force memory writes to complete before letting h/w
3032 * know there are new descriptors to fetch. (Only
3033 * applicable for weak-ordered memory model archs,
3034 * such as IA-64).
3035 */
3036 wmb();
3037
3038 tx_ring->tx_buffer_info[first].next_to_watch = tx_desc;
92915f71 3039 tx_ring->next_to_use = i;
92915f71
GR		 3040}
3041
fb40195c 3042static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 3043{
fb40195c 3044 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
92915f71
GR		 3045 /* Herbert's original patch had:
3046 * smp_mb__after_netif_stop_queue();
3047 * but since that doesn't exist yet, just open code it. */
3048 smp_mb();
3049
3050 /* We need to check again in a case another CPU has just
3051 * made room available. */
f880d07b 3052 if (likely(ixgbevf_desc_unused(tx_ring) < size))
92915f71
GR		 3053 return -EBUSY;
3054
3055 /* A reprieve! - use start_queue because it doesn't call schedule */
fb40195c 3056 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
095e2617
ET		 3057 ++tx_ring->tx_stats.restart_queue;
3058
92915f71
GR		 3059 return 0;
3060}
3061
fb40195c 3062static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 3063{
f880d07b 3064 if (likely(ixgbevf_desc_unused(tx_ring) >= size))
92915f71 3065 return 0;
fb40195c 3066 return __ixgbevf_maybe_stop_tx(tx_ring, size);
92915f71
GR		 3067}
3068
3069static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3070{
3071 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3072 struct ixgbevf_ring *tx_ring;
3073 unsigned int first;
3074 unsigned int tx_flags = 0;
3075 u8 hdr_len = 0;
3076 int r_idx = 0, tso;
3595990a
AD		 3077 u16 count = TXD_USE_COUNT(skb_headlen(skb));
3078#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3079 unsigned short f;
3080#endif
f9d08f16 3081 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
46acc460 3082 if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
f9d08f16
GR		 3083 dev_kfree_skb(skb);
3084 return NETDEV_TX_OK;
3085 }
92915f71 3086
87e70ab9 3087 tx_ring = adapter->tx_ring[r_idx];
92915f71 3088
3595990a
AD		 3089 /*
3090 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3091 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3092 * + 2 desc gap to keep tail from touching head,
3093 * + 1 desc for context descriptor,
3094 * otherwise try next time
3095 */
3096#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3097 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3098 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3099#else
3100 count += skb_shinfo(skb)->nr_frags;
3101#endif
fb40195c 3102 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
095e2617 3103 tx_ring->tx_stats.tx_busy++;
3595990a
AD		 3104 return NETDEV_TX_BUSY;
3105 }
3106
eab6d18d 3107 if (vlan_tx_tag_present(skb)) {
92915f71
GR		 3108 tx_flags |= vlan_tx_tag_get(skb);
3109 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3110 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3111 }
3112
92915f71
GR		 3113 first = tx_ring->next_to_use;
3114
3115 if (skb->protocol == htons(ETH_P_IP))
3116 tx_flags |= IXGBE_TX_FLAGS_IPV4;
70a10e25 3117 tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len);
92915f71
GR		 3118 if (tso < 0) {
3119 dev_kfree_skb_any(skb);
3120 return NETDEV_TX_OK;
3121 }
3122
3123 if (tso)
70a10e25
AD		 3124 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM;
3125 else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags))
92915f71
GR		 3126 tx_flags |= IXGBE_TX_FLAGS_CSUM;
3127
70a10e25 3128 ixgbevf_tx_queue(tx_ring, tx_flags,
e757e3e1
AD		 3129 ixgbevf_tx_map(tx_ring, skb, tx_flags),
3130 first, skb->len, hdr_len);
70a10e25 3131
5cdab2f6 3132 writel(tx_ring->next_to_use, tx_ring->tail);
92915f71 3133
fb40195c 3134 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
92915f71
GR		 3135
3136 return NETDEV_TX_OK;
3137}
3138
92915f71
GR		 3139/**
3140 * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3141 * @netdev: network interface device structure
3142 * @p: pointer to an address structure
3143 *
3144 * Returns 0 on success, negative on failure
3145 **/
3146static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3147{
3148 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3149 struct ixgbe_hw *hw = &adapter->hw;
3150 struct sockaddr *addr = p;
3151
3152 if (!is_valid_ether_addr(addr->sa_data))
3153 return -EADDRNOTAVAIL;
3154
3155 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3156 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3157
55fdd45b 3158 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 3159
92fe0bf7 3160 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
92915f71 3161
55fdd45b 3162 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 3163
92915f71
GR		 3164 return 0;
3165}
3166
3167/**
3168 * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3169 * @netdev: network interface device structure
3170 * @new_mtu: new value for maximum frame size
3171 *
3172 * Returns 0 on success, negative on failure
3173 **/
3174static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3175{
3176 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3177 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
69bfbec4 3178 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
69bfbec4 3179
56e94095
AD		 3180 switch (adapter->hw.api_version) {
3181 case ixgbe_mbox_api_11:
69bfbec4 3182 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
56e94095
AD		 3183 break;
3184 default:
3185 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3186 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3187 break;
3188 }
92915f71
GR		 3189
3190 /* MTU < 68 is an error and causes problems on some kernels */
69bfbec4 3191 if ((new_mtu < 68) || (max_frame > max_possible_frame))
92915f71
GR		 3192 return -EINVAL;
3193
3194 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3195 netdev->mtu, new_mtu);
3196 /* must set new MTU before calling down or up */
3197 netdev->mtu = new_mtu;
3198
3199 if (netif_running(netdev))
3200 ixgbevf_reinit_locked(adapter);
3201
3202 return 0;
3203}
3204
0ac1e8ce 3205static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
92915f71
GR		 3206{
3207 struct net_device *netdev = pci_get_drvdata(pdev);
3208 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3209#ifdef CONFIG_PM
3210 int retval = 0;
3211#endif
92915f71
GR		 3212
3213 netif_device_detach(netdev);
3214
3215 if (netif_running(netdev)) {
0ac1e8ce 3216 rtnl_lock();
92915f71
GR		 3217 ixgbevf_down(adapter);
3218 ixgbevf_free_irq(adapter);
3219 ixgbevf_free_all_tx_resources(adapter);
3220 ixgbevf_free_all_rx_resources(adapter);
0ac1e8ce 3221 rtnl_unlock();
92915f71
GR		 3222 }
3223
0ac1e8ce 3224 ixgbevf_clear_interrupt_scheme(adapter);
92915f71 3225
0ac1e8ce
AD		 3226#ifdef CONFIG_PM
3227 retval = pci_save_state(pdev);
3228 if (retval)
3229 return retval;
92915f71 3230
0ac1e8ce 3231#endif
92915f71 3232 pci_disable_device(pdev);
0ac1e8ce
AD		 3233
3234 return 0;
3235}
3236
3237#ifdef CONFIG_PM
3238static int ixgbevf_resume(struct pci_dev *pdev)
3239{
27ae2967
WY		 3240 struct net_device *netdev = pci_get_drvdata(pdev);
3241 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3242 u32 err;
3243
3244 pci_set_power_state(pdev, PCI_D0);
3245 pci_restore_state(pdev);
3246 /*
3247 * pci_restore_state clears dev->state_saved so call
3248 * pci_save_state to restore it.
3249 */
3250 pci_save_state(pdev);
3251
3252 err = pci_enable_device_mem(pdev);
3253 if (err) {
3254 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3255 return err;
3256 }
3257 pci_set_master(pdev);
3258
798e381a
DS		 3259 ixgbevf_reset(adapter);
3260
0ac1e8ce
AD		 3261 rtnl_lock();
3262 err = ixgbevf_init_interrupt_scheme(adapter);
3263 rtnl_unlock();
3264 if (err) {
3265 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3266 return err;
3267 }
3268
0ac1e8ce
AD		 3269 if (netif_running(netdev)) {
3270 err = ixgbevf_open(netdev);
3271 if (err)
3272 return err;
3273 }
3274
3275 netif_device_attach(netdev);
3276
3277 return err;
3278}
3279
3280#endif /* CONFIG_PM */
3281static void ixgbevf_shutdown(struct pci_dev *pdev)
3282{
3283 ixgbevf_suspend(pdev, PMSG_SUSPEND);
92915f71
GR		 3284}
3285
4197aa7b
ED		 3286static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3287 struct rtnl_link_stats64 *stats)
3288{
3289 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3290 unsigned int start;
3291 u64 bytes, packets;
3292 const struct ixgbevf_ring *ring;
3293 int i;
3294
3295 ixgbevf_update_stats(adapter);
3296
3297 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3298
3299 for (i = 0; i < adapter->num_rx_queues; i++) {
87e70ab9 3300 ring = adapter->rx_ring[i];
4197aa7b
ED		 3301 do {
3302 start = u64_stats_fetch_begin_bh(&ring->syncp);
095e2617
ET		 3303 bytes = ring->stats.bytes;
3304 packets = ring->stats.packets;
4197aa7b
ED		 3305 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3306 stats->rx_bytes += bytes;
3307 stats->rx_packets += packets;
3308 }
3309
3310 for (i = 0; i < adapter->num_tx_queues; i++) {
87e70ab9 3311 ring = adapter->tx_ring[i];
4197aa7b
ED		 3312 do {
3313 start = u64_stats_fetch_begin_bh(&ring->syncp);
095e2617
ET		 3314 bytes = ring->stats.bytes;
3315 packets = ring->stats.packets;
4197aa7b
ED		 3316 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3317 stats->tx_bytes += bytes;
3318 stats->tx_packets += packets;
3319 }
3320
3321 return stats;
3322}
3323
0ac1e8ce 3324static const struct net_device_ops ixgbevf_netdev_ops = {
c12db769
SH		 3325 .ndo_open = ixgbevf_open,
3326 .ndo_stop = ixgbevf_close,
3327 .ndo_start_xmit = ixgbevf_xmit_frame,
3328 .ndo_set_rx_mode = ixgbevf_set_rx_mode,
4197aa7b 3329 .ndo_get_stats64 = ixgbevf_get_stats,
92915f71 3330 .ndo_validate_addr = eth_validate_addr,
c12db769
SH		 3331 .ndo_set_mac_address = ixgbevf_set_mac,
3332 .ndo_change_mtu = ixgbevf_change_mtu,
3333 .ndo_tx_timeout = ixgbevf_tx_timeout,
c12db769
SH		 3334 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid,
3335 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid,
c777cdfa
JK		 3336#ifdef CONFIG_NET_RX_BUSY_POLL
3337 .ndo_busy_poll = ixgbevf_busy_poll_recv,
3338#endif
92915f71 3339};
92915f71
GR		 3340
3341static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3342{
0ac1e8ce 3343 dev->netdev_ops = &ixgbevf_netdev_ops;
92915f71
GR		 3344 ixgbevf_set_ethtool_ops(dev);
3345 dev->watchdog_timeo = 5 * HZ;
3346}
3347
3348/**
3349 * ixgbevf_probe - Device Initialization Routine
3350 * @pdev: PCI device information struct
3351 * @ent: entry in ixgbevf_pci_tbl
3352 *
3353 * Returns 0 on success, negative on failure
3354 *
3355 * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3356 * The OS initialization, configuring of the adapter private structure,
3357 * and a hardware reset occur.
3358 **/
1dd06ae8 3359static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
92915f71
GR		 3360{
3361 struct net_device *netdev;
3362 struct ixgbevf_adapter *adapter = NULL;
3363 struct ixgbe_hw *hw = NULL;
3364 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3365 static int cards_found;
3366 int err, pci_using_dac;
3367
3368 err = pci_enable_device(pdev);
3369 if (err)
3370 return err;
3371
53567aa4 3372 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
92915f71
GR		 3373 pci_using_dac = 1;
3374 } else {
53567aa4 3375 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
92915f71 3376 if (err) {
53567aa4
RK		 3377 dev_err(&pdev->dev, "No usable DMA "
3378 "configuration, aborting\n");
3379 goto err_dma;
92915f71
GR		 3380 }
3381 pci_using_dac = 0;
3382 }
3383
3384 err = pci_request_regions(pdev, ixgbevf_driver_name);
3385 if (err) {
3386 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3387 goto err_pci_reg;
3388 }
3389
3390 pci_set_master(pdev);
3391
92915f71
GR		 3392 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3393 MAX_TX_QUEUES);
92915f71
GR		 3394 if (!netdev) {
3395 err = -ENOMEM;
3396 goto err_alloc_etherdev;
3397 }
3398
3399 SET_NETDEV_DEV(netdev, &pdev->dev);
3400
3401 pci_set_drvdata(pdev, netdev);
3402 adapter = netdev_priv(netdev);
3403
3404 adapter->netdev = netdev;
3405 adapter->pdev = pdev;
3406 hw = &adapter->hw;
3407 hw->back = adapter;
b3f4d599 3408 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
92915f71
GR		 3409
3410 /*
3411 * call save state here in standalone driver because it relies on
3412 * adapter struct to exist, and needs to call netdev_priv
3413 */
3414 pci_save_state(pdev);
3415
3416 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3417 pci_resource_len(pdev, 0));
3418 if (!hw->hw_addr) {
3419 err = -EIO;
3420 goto err_ioremap;
3421 }
3422
3423 ixgbevf_assign_netdev_ops(netdev);
3424
3425 adapter->bd_number = cards_found;
3426
3427 /* Setup hw api */
3428 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3429 hw->mac.type = ii->mac;
3430
3431 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
f416dfc0 3432 sizeof(struct ixgbe_mbx_operations));
92915f71 3433
92915f71
GR		 3434 /* setup the private structure */
3435 err = ixgbevf_sw_init(adapter);
1a0d6ae5
DK		 3436 if (err)
3437 goto err_sw_init;
3438
3439 /* The HW MAC address was set and/or determined in sw_init */
1a0d6ae5
DK		 3440 if (!is_valid_ether_addr(netdev->dev_addr)) {
3441 pr_err("invalid MAC address\n");
3442 err = -EIO;
3443 goto err_sw_init;
3444 }
92915f71 3445
471a76de 3446 netdev->hw_features = NETIF_F_SG |
92915f71 3447 NETIF_F_IP_CSUM |
471a76de
MM		 3448 NETIF_F_IPV6_CSUM |
3449 NETIF_F_TSO |
3450 NETIF_F_TSO6 |
3451 NETIF_F_RXCSUM;
3452
3453 netdev->features = netdev->hw_features |
f646968f
PM		 3454 NETIF_F_HW_VLAN_CTAG_TX |
3455 NETIF_F_HW_VLAN_CTAG_RX |
3456 NETIF_F_HW_VLAN_CTAG_FILTER;
92915f71 3457
92915f71
GR		 3458 netdev->vlan_features |= NETIF_F_TSO;
3459 netdev->vlan_features |= NETIF_F_TSO6;
3460 netdev->vlan_features |= NETIF_F_IP_CSUM;
3bfacf96 3461 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
92915f71
GR		 3462 netdev->vlan_features |= NETIF_F_SG;
3463
3464 if (pci_using_dac)
3465 netdev->features |= NETIF_F_HIGHDMA;
3466
01789349
JP		 3467 netdev->priv_flags |= IFF_UNICAST_FLT;
3468
92915f71 3469 init_timer(&adapter->watchdog_timer);
c061b18d 3470 adapter->watchdog_timer.function = ixgbevf_watchdog;
92915f71
GR		 3471 adapter->watchdog_timer.data = (unsigned long)adapter;
3472
3473 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3474 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3475
3476 err = ixgbevf_init_interrupt_scheme(adapter);
3477 if (err)
3478 goto err_sw_init;
3479
92915f71
GR		 3480 strcpy(netdev->name, "eth%d");
3481
3482 err = register_netdev(netdev);
3483 if (err)
3484 goto err_register;
3485
5d426ad1
GR		 3486 netif_carrier_off(netdev);
3487
33bd9f60
GR		 3488 ixgbevf_init_last_counter_stats(adapter);
3489
92915f71 3490 /* print the MAC address */
f794e7ef 3491 hw_dbg(hw, "%pM\n", netdev->dev_addr);
92915f71
GR		 3492
3493 hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3494
92915f71
GR		 3495 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3496 cards_found++;
3497 return 0;
3498
3499err_register:
0ac1e8ce 3500 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3501err_sw_init:
3502 ixgbevf_reset_interrupt_capability(adapter);
3503 iounmap(hw->hw_addr);
3504err_ioremap:
3505 free_netdev(netdev);
3506err_alloc_etherdev:
3507 pci_release_regions(pdev);
3508err_pci_reg:
3509err_dma:
3510 pci_disable_device(pdev);
3511 return err;
3512}
3513
3514/**
3515 * ixgbevf_remove - Device Removal Routine
3516 * @pdev: PCI device information struct
3517 *
3518 * ixgbevf_remove is called by the PCI subsystem to alert the driver
3519 * that it should release a PCI device. The could be caused by a
3520 * Hot-Plug event, or because the driver is going to be removed from
3521 * memory.
3522 **/
9f9a12f8 3523static void ixgbevf_remove(struct pci_dev *pdev)
92915f71
GR		 3524{
3525 struct net_device *netdev = pci_get_drvdata(pdev);
3526 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3527
3528 set_bit(__IXGBEVF_DOWN, &adapter->state);
3529
3530 del_timer_sync(&adapter->watchdog_timer);
3531
23f333a2 3532 cancel_work_sync(&adapter->reset_task);
92915f71
GR		 3533 cancel_work_sync(&adapter->watchdog_task);
3534
fd13a9ab 3535 if (netdev->reg_state == NETREG_REGISTERED)
92915f71 3536 unregister_netdev(netdev);
92915f71 3537
0ac1e8ce 3538 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3539 ixgbevf_reset_interrupt_capability(adapter);
3540
3541 iounmap(adapter->hw.hw_addr);
3542 pci_release_regions(pdev);
3543
3544 hw_dbg(&adapter->hw, "Remove complete\n");
3545
92915f71
GR		 3546 free_netdev(netdev);
3547
3548 pci_disable_device(pdev);
3549}
3550
9f19f31d
AD		 3551/**
3552 * ixgbevf_io_error_detected - called when PCI error is detected
3553 * @pdev: Pointer to PCI device
3554 * @state: The current pci connection state
3555 *
3556 * This function is called after a PCI bus error affecting
3557 * this device has been detected.
3558 */
3559static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3560 pci_channel_state_t state)
3561{
3562 struct net_device *netdev = pci_get_drvdata(pdev);
3563 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3564
3565 netif_device_detach(netdev);
3566
3567 if (state == pci_channel_io_perm_failure)
3568 return PCI_ERS_RESULT_DISCONNECT;
3569
3570 if (netif_running(netdev))
3571 ixgbevf_down(adapter);
3572
3573 pci_disable_device(pdev);
3574
3575 /* Request a slot slot reset. */
3576 return PCI_ERS_RESULT_NEED_RESET;
3577}
3578
3579/**
3580 * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3581 * @pdev: Pointer to PCI device
3582 *
3583 * Restart the card from scratch, as if from a cold-boot. Implementation
3584 * resembles the first-half of the ixgbevf_resume routine.
3585 */
3586static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3587{
3588 struct net_device *netdev = pci_get_drvdata(pdev);
3589 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3590
3591 if (pci_enable_device_mem(pdev)) {
3592 dev_err(&pdev->dev,
3593 "Cannot re-enable PCI device after reset.\n");
3594 return PCI_ERS_RESULT_DISCONNECT;
3595 }
3596
3597 pci_set_master(pdev);
3598
3599 ixgbevf_reset(adapter);
3600
3601 return PCI_ERS_RESULT_RECOVERED;
3602}
3603
3604/**
3605 * ixgbevf_io_resume - called when traffic can start flowing again.
3606 * @pdev: Pointer to PCI device
3607 *
3608 * This callback is called when the error recovery driver tells us that
3609 * its OK to resume normal operation. Implementation resembles the
3610 * second-half of the ixgbevf_resume routine.
3611 */
3612static void ixgbevf_io_resume(struct pci_dev *pdev)
3613{
3614 struct net_device *netdev = pci_get_drvdata(pdev);
3615 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3616
3617 if (netif_running(netdev))
3618 ixgbevf_up(adapter);
3619
3620 netif_device_attach(netdev);
3621}
3622
3623/* PCI Error Recovery (ERS) */
3646f0e5 3624static const struct pci_error_handlers ixgbevf_err_handler = {
9f19f31d
AD		 3625 .error_detected = ixgbevf_io_error_detected,
3626 .slot_reset = ixgbevf_io_slot_reset,
3627 .resume = ixgbevf_io_resume,
3628};
3629
92915f71
GR		 3630static struct pci_driver ixgbevf_driver = {
3631 .name = ixgbevf_driver_name,
3632 .id_table = ixgbevf_pci_tbl,
3633 .probe = ixgbevf_probe,
9f9a12f8 3634 .remove = ixgbevf_remove,
0ac1e8ce
AD		 3635#ifdef CONFIG_PM
3636 /* Power Management Hooks */
3637 .suspend = ixgbevf_suspend,
3638 .resume = ixgbevf_resume,
3639#endif
92915f71 3640 .shutdown = ixgbevf_shutdown,
9f19f31d 3641 .err_handler = &ixgbevf_err_handler
92915f71
GR		 3642};
3643
3644/**
65d676c8 3645 * ixgbevf_init_module - Driver Registration Routine
92915f71 3646 *
65d676c8 3647 * ixgbevf_init_module is the first routine called when the driver is
92915f71
GR		 3648 * loaded. All it does is register with the PCI subsystem.
3649 **/
3650static int __init ixgbevf_init_module(void)
3651{
3652 int ret;
dbd9636e
JK		 3653 pr_info("%s - version %s\n", ixgbevf_driver_string,
3654 ixgbevf_driver_version);
92915f71 3655
dbd9636e 3656 pr_info("%s\n", ixgbevf_copyright);
92915f71
GR		 3657
3658 ret = pci_register_driver(&ixgbevf_driver);
3659 return ret;
3660}
3661
3662module_init(ixgbevf_init_module);
3663
3664/**
65d676c8 3665 * ixgbevf_exit_module - Driver Exit Cleanup Routine
92915f71 3666 *
65d676c8 3667 * ixgbevf_exit_module is called just before the driver is removed
92915f71
GR		 3668 * from memory.
3669 **/
3670static void __exit ixgbevf_exit_module(void)
3671{
3672 pci_unregister_driver(&ixgbevf_driver);
3673}
3674
3675#ifdef DEBUG
3676/**
65d676c8 3677 * ixgbevf_get_hw_dev_name - return device name string
92915f71
GR		 3678 * used by hardware layer to print debugging information
3679 **/
3680char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3681{
3682 struct ixgbevf_adapter *adapter = hw->back;
3683 return adapter->netdev->name;
3684}
3685
3686#endif
3687module_exit(ixgbevf_exit_module);
3688
3689/* ixgbevf_main.c */
Linux kernel - Deliverables
This page took 0.687548 seconds and 5 git commands to generate.