projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'acpica-gpe' into release
[deliverable/linux.git] / drivers / net / igb / igb_main.c
1 /*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2009 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26 *******************************************************************************/
27
28 #include <linux/module.h>
29 #include <linux/types.h>
30 #include <linux/init.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/netdevice.h>
34 #include <linux/ipv6.h>
35 #include <linux/slab.h>
36 #include <net/checksum.h>
37 #include <net/ip6_checksum.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/mii.h>
40 #include <linux/ethtool.h>
41 #include <linux/if_vlan.h>
42 #include <linux/pci.h>
43 #include <linux/pci-aspm.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/if_ether.h>
47 #include <linux/aer.h>
48 #ifdef CONFIG_IGB_DCA
49 #include <linux/dca.h>
50 #endif
51 #include "igb.h"
52
53 #define DRV_VERSION "2.1.0-k2"
54 char igb_driver_name[] = "igb";
55 char igb_driver_version[] = DRV_VERSION;
56 static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
58 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
59
60 static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62 };
63
64 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
85 /* required last entry */
86 {0, }
87 };
88
89 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
90
91 void igb_reset(struct igb_adapter *);
92 static int igb_setup_all_tx_resources(struct igb_adapter *);
93 static int igb_setup_all_rx_resources(struct igb_adapter *);
94 static void igb_free_all_tx_resources(struct igb_adapter *);
95 static void igb_free_all_rx_resources(struct igb_adapter *);
96 static void igb_setup_mrqc(struct igb_adapter *);
97 void igb_update_stats(struct igb_adapter *);
98 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
99 static void __devexit igb_remove(struct pci_dev *pdev);
100 static int igb_sw_init(struct igb_adapter *);
101 static int igb_open(struct net_device *);
102 static int igb_close(struct net_device *);
103 static void igb_configure_tx(struct igb_adapter *);
104 static void igb_configure_rx(struct igb_adapter *);
105 static void igb_clean_all_tx_rings(struct igb_adapter *);
106 static void igb_clean_all_rx_rings(struct igb_adapter *);
107 static void igb_clean_tx_ring(struct igb_ring *);
108 static void igb_clean_rx_ring(struct igb_ring *);
109 static void igb_set_rx_mode(struct net_device *);
110 static void igb_update_phy_info(unsigned long);
111 static void igb_watchdog(unsigned long);
112 static void igb_watchdog_task(struct work_struct *);
113 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
114 static struct net_device_stats *igb_get_stats(struct net_device *);
115 static int igb_change_mtu(struct net_device *, int);
116 static int igb_set_mac(struct net_device *, void *);
117 static void igb_set_uta(struct igb_adapter *adapter);
118 static irqreturn_t igb_intr(int irq, void *);
119 static irqreturn_t igb_intr_msi(int irq, void *);
120 static irqreturn_t igb_msix_other(int irq, void *);
121 static irqreturn_t igb_msix_ring(int irq, void *);
122 #ifdef CONFIG_IGB_DCA
123 static void igb_update_dca(struct igb_q_vector *);
124 static void igb_setup_dca(struct igb_adapter *);
125 #endif /* CONFIG_IGB_DCA */
126 static bool igb_clean_tx_irq(struct igb_q_vector *);
127 static int igb_poll(struct napi_struct *, int);
128 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
129 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
130 static void igb_tx_timeout(struct net_device *);
131 static void igb_reset_task(struct work_struct *);
132 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
133 static void igb_vlan_rx_add_vid(struct net_device *, u16);
134 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
135 static void igb_restore_vlan(struct igb_adapter *);
136 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
137 static void igb_ping_all_vfs(struct igb_adapter *);
138 static void igb_msg_task(struct igb_adapter *);
139 static void igb_vmm_control(struct igb_adapter *);
140 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
141 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
142 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
143 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
144 int vf, u16 vlan, u8 qos);
145 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
146 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
147 struct ifla_vf_info *ivi);
148
149 #ifdef CONFIG_PM
150 static int igb_suspend(struct pci_dev *, pm_message_t);
151 static int igb_resume(struct pci_dev *);
152 #endif
153 static void igb_shutdown(struct pci_dev *);
154 #ifdef CONFIG_IGB_DCA
155 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
156 static struct notifier_block dca_notifier = {
157 .notifier_call = igb_notify_dca,
158 .next = NULL,
159 .priority = 0
160 };
161 #endif
162 #ifdef CONFIG_NET_POLL_CONTROLLER
163 /* for netdump / net console */
164 static void igb_netpoll(struct net_device *);
165 #endif
166 #ifdef CONFIG_PCI_IOV
167 static unsigned int max_vfs = 0;
168 module_param(max_vfs, uint, 0);
169 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
170 "per physical function");
171 #endif /* CONFIG_PCI_IOV */
172
173 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
174 pci_channel_state_t);
175 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
176 static void igb_io_resume(struct pci_dev *);
177
178 static struct pci_error_handlers igb_err_handler = {
179 .error_detected = igb_io_error_detected,
180 .slot_reset = igb_io_slot_reset,
181 .resume = igb_io_resume,
182 };
183
184
185 static struct pci_driver igb_driver = {
186 .name = igb_driver_name,
187 .id_table = igb_pci_tbl,
188 .probe = igb_probe,
189 .remove = __devexit_p(igb_remove),
190 #ifdef CONFIG_PM
191 /* Power Managment Hooks */
192 .suspend = igb_suspend,
193 .resume = igb_resume,
194 #endif
195 .shutdown = igb_shutdown,
196 .err_handler = &igb_err_handler
197 };
198
199 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
200 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
201 MODULE_LICENSE("GPL");
202 MODULE_VERSION(DRV_VERSION);
203
204 struct igb_reg_info {
205 u32 ofs;
206 char *name;
207 };
208
209 static const struct igb_reg_info igb_reg_info_tbl[] = {
210
211 /* General Registers */
212 {E1000_CTRL, "CTRL"},
213 {E1000_STATUS, "STATUS"},
214 {E1000_CTRL_EXT, "CTRL_EXT"},
215
216 /* Interrupt Registers */
217 {E1000_ICR, "ICR"},
218
219 /* RX Registers */
220 {E1000_RCTL, "RCTL"},
221 {E1000_RDLEN(0), "RDLEN"},
222 {E1000_RDH(0), "RDH"},
223 {E1000_RDT(0), "RDT"},
224 {E1000_RXDCTL(0), "RXDCTL"},
225 {E1000_RDBAL(0), "RDBAL"},
226 {E1000_RDBAH(0), "RDBAH"},
227
228 /* TX Registers */
229 {E1000_TCTL, "TCTL"},
230 {E1000_TDBAL(0), "TDBAL"},
231 {E1000_TDBAH(0), "TDBAH"},
232 {E1000_TDLEN(0), "TDLEN"},
233 {E1000_TDH(0), "TDH"},
234 {E1000_TDT(0), "TDT"},
235 {E1000_TXDCTL(0), "TXDCTL"},
236 {E1000_TDFH, "TDFH"},
237 {E1000_TDFT, "TDFT"},
238 {E1000_TDFHS, "TDFHS"},
239 {E1000_TDFPC, "TDFPC"},
240
241 /* List Terminator */
242 {}
243 };
244
245 /*
246 * igb_regdump - register printout routine
247 */
248 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
249 {
250 int n = 0;
251 char rname[16];
252 u32 regs[8];
253
254 switch (reginfo->ofs) {
255 case E1000_RDLEN(0):
256 for (n = 0; n < 4; n++)
257 regs[n] = rd32(E1000_RDLEN(n));
258 break;
259 case E1000_RDH(0):
260 for (n = 0; n < 4; n++)
261 regs[n] = rd32(E1000_RDH(n));
262 break;
263 case E1000_RDT(0):
264 for (n = 0; n < 4; n++)
265 regs[n] = rd32(E1000_RDT(n));
266 break;
267 case E1000_RXDCTL(0):
268 for (n = 0; n < 4; n++)
269 regs[n] = rd32(E1000_RXDCTL(n));
270 break;
271 case E1000_RDBAL(0):
272 for (n = 0; n < 4; n++)
273 regs[n] = rd32(E1000_RDBAL(n));
274 break;
275 case E1000_RDBAH(0):
276 for (n = 0; n < 4; n++)
277 regs[n] = rd32(E1000_RDBAH(n));
278 break;
279 case E1000_TDBAL(0):
280 for (n = 0; n < 4; n++)
281 regs[n] = rd32(E1000_RDBAL(n));
282 break;
283 case E1000_TDBAH(0):
284 for (n = 0; n < 4; n++)
285 regs[n] = rd32(E1000_TDBAH(n));
286 break;
287 case E1000_TDLEN(0):
288 for (n = 0; n < 4; n++)
289 regs[n] = rd32(E1000_TDLEN(n));
290 break;
291 case E1000_TDH(0):
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_TDH(n));
294 break;
295 case E1000_TDT(0):
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_TDT(n));
298 break;
299 case E1000_TXDCTL(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_TXDCTL(n));
302 break;
303 default:
304 printk(KERN_INFO "%-15s %08x\n",
305 reginfo->name, rd32(reginfo->ofs));
306 return;
307 }
308
309 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
310 printk(KERN_INFO "%-15s ", rname);
311 for (n = 0; n < 4; n++)
312 printk(KERN_CONT "%08x ", regs[n]);
313 printk(KERN_CONT "\n");
314 }
315
316 /*
317 * igb_dump - Print registers, tx-rings and rx-rings
318 */
319 static void igb_dump(struct igb_adapter *adapter)
320 {
321 struct net_device *netdev = adapter->netdev;
322 struct e1000_hw *hw = &adapter->hw;
323 struct igb_reg_info *reginfo;
324 int n = 0;
325 struct igb_ring *tx_ring;
326 union e1000_adv_tx_desc *tx_desc;
327 struct my_u0 { u64 a; u64 b; } *u0;
328 struct igb_buffer *buffer_info;
329 struct igb_ring *rx_ring;
330 union e1000_adv_rx_desc *rx_desc;
331 u32 staterr;
332 int i = 0;
333
334 if (!netif_msg_hw(adapter))
335 return;
336
337 /* Print netdevice Info */
338 if (netdev) {
339 dev_info(&adapter->pdev->dev, "Net device Info\n");
340 printk(KERN_INFO "Device Name state "
341 "trans_start last_rx\n");
342 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
343 netdev->name,
344 netdev->state,
345 netdev->trans_start,
346 netdev->last_rx);
347 }
348
349 /* Print Registers */
350 dev_info(&adapter->pdev->dev, "Register Dump\n");
351 printk(KERN_INFO " Register Name Value\n");
352 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
353 reginfo->name; reginfo++) {
354 igb_regdump(hw, reginfo);
355 }
356
357 /* Print TX Ring Summary */
358 if (!netdev || !netif_running(netdev))
359 goto exit;
360
361 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
362 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
363 " leng ntw timestamp\n");
364 for (n = 0; n < adapter->num_tx_queues; n++) {
365 tx_ring = adapter->tx_ring[n];
366 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
367 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
368 n, tx_ring->next_to_use, tx_ring->next_to_clean,
369 (u64)buffer_info->dma,
370 buffer_info->length,
371 buffer_info->next_to_watch,
372 (u64)buffer_info->time_stamp);
373 }
374
375 /* Print TX Rings */
376 if (!netif_msg_tx_done(adapter))
377 goto rx_ring_summary;
378
379 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
380
381 /* Transmit Descriptor Formats
382 *
383 * Advanced Transmit Descriptor
384 * +--------------------------------------------------------------+
385 * 0 | Buffer Address [63:0] |
386 * +--------------------------------------------------------------+
387 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
388 * +--------------------------------------------------------------+
389 * 63 46 45 40 39 38 36 35 32 31 24 15 0
390 */
391
392 for (n = 0; n < adapter->num_tx_queues; n++) {
393 tx_ring = adapter->tx_ring[n];
394 printk(KERN_INFO "------------------------------------\n");
395 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "T [desc] [address 63:0 ] "
398 "[PlPOCIStDDM Ln] [bi->dma ] "
399 "leng ntw timestamp bi->skb\n");
400
401 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
402 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
403 buffer_info = &tx_ring->buffer_info[i];
404 u0 = (struct my_u0 *)tx_desc;
405 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
406 " %04X %3X %016llX %p", i,
407 le64_to_cpu(u0->a),
408 le64_to_cpu(u0->b),
409 (u64)buffer_info->dma,
410 buffer_info->length,
411 buffer_info->next_to_watch,
412 (u64)buffer_info->time_stamp,
413 buffer_info->skb);
414 if (i == tx_ring->next_to_use &&
415 i == tx_ring->next_to_clean)
416 printk(KERN_CONT " NTC/U\n");
417 else if (i == tx_ring->next_to_use)
418 printk(KERN_CONT " NTU\n");
419 else if (i == tx_ring->next_to_clean)
420 printk(KERN_CONT " NTC\n");
421 else
422 printk(KERN_CONT "\n");
423
424 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
425 print_hex_dump(KERN_INFO, "",
426 DUMP_PREFIX_ADDRESS,
427 16, 1, phys_to_virt(buffer_info->dma),
428 buffer_info->length, true);
429 }
430 }
431
432 /* Print RX Rings Summary */
433 rx_ring_summary:
434 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
435 printk(KERN_INFO "Queue [NTU] [NTC]\n");
436 for (n = 0; n < adapter->num_rx_queues; n++) {
437 rx_ring = adapter->rx_ring[n];
438 printk(KERN_INFO " %5d %5X %5X\n", n,
439 rx_ring->next_to_use, rx_ring->next_to_clean);
440 }
441
442 /* Print RX Rings */
443 if (!netif_msg_rx_status(adapter))
444 goto exit;
445
446 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
447
448 /* Advanced Receive Descriptor (Read) Format
449 * 63 1 0
450 * +-----------------------------------------------------+
451 * 0 | Packet Buffer Address [63:1] |A0/NSE|
452 * +----------------------------------------------+------+
453 * 8 | Header Buffer Address [63:1] | DD |
454 * +-----------------------------------------------------+
455 *
456 *
457 * Advanced Receive Descriptor (Write-Back) Format
458 *
459 * 63 48 47 32 31 30 21 20 17 16 4 3 0
460 * +------------------------------------------------------+
461 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
462 * | Checksum Ident | | | | Type | Type |
463 * +------------------------------------------------------+
464 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
465 * +------------------------------------------------------+
466 * 63 48 47 32 31 20 19 0
467 */
468
469 for (n = 0; n < adapter->num_rx_queues; n++) {
470 rx_ring = adapter->rx_ring[n];
471 printk(KERN_INFO "------------------------------------\n");
472 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "R [desc] [ PktBuf A0] "
475 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
476 "<-- Adv Rx Read format\n");
477 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
478 "[vl er S cks ln] ---------------- [bi->skb] "
479 "<-- Adv Rx Write-Back format\n");
480
481 for (i = 0; i < rx_ring->count; i++) {
482 buffer_info = &rx_ring->buffer_info[i];
483 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
484 u0 = (struct my_u0 *)rx_desc;
485 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
486 if (staterr & E1000_RXD_STAT_DD) {
487 /* Descriptor Done */
488 printk(KERN_INFO "RWB[0x%03X] %016llX "
489 "%016llX ---------------- %p", i,
490 le64_to_cpu(u0->a),
491 le64_to_cpu(u0->b),
492 buffer_info->skb);
493 } else {
494 printk(KERN_INFO "R [0x%03X] %016llX "
495 "%016llX %016llX %p", i,
496 le64_to_cpu(u0->a),
497 le64_to_cpu(u0->b),
498 (u64)buffer_info->dma,
499 buffer_info->skb);
500
501 if (netif_msg_pktdata(adapter)) {
502 print_hex_dump(KERN_INFO, "",
503 DUMP_PREFIX_ADDRESS,
504 16, 1,
505 phys_to_virt(buffer_info->dma),
506 rx_ring->rx_buffer_len, true);
507 if (rx_ring->rx_buffer_len
508 < IGB_RXBUFFER_1024)
509 print_hex_dump(KERN_INFO, "",
510 DUMP_PREFIX_ADDRESS,
511 16, 1,
512 phys_to_virt(
513 buffer_info->page_dma +
514 buffer_info->page_offset),
515 PAGE_SIZE/2, true);
516 }
517 }
518
519 if (i == rx_ring->next_to_use)
520 printk(KERN_CONT " NTU\n");
521 else if (i == rx_ring->next_to_clean)
522 printk(KERN_CONT " NTC\n");
523 else
524 printk(KERN_CONT "\n");
525
526 }
527 }
528
529 exit:
530 return;
531 }
532
533
534 /**
535 * igb_read_clock - read raw cycle counter (to be used by time counter)
536 */
537 static cycle_t igb_read_clock(const struct cyclecounter *tc)
538 {
539 struct igb_adapter *adapter =
540 container_of(tc, struct igb_adapter, cycles);
541 struct e1000_hw *hw = &adapter->hw;
542 u64 stamp = 0;
543 int shift = 0;
544
545 /*
546 * The timestamp latches on lowest register read. For the 82580
547 * the lowest register is SYSTIMR instead of SYSTIML. However we never
548 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
549 */
550 if (hw->mac.type == e1000_82580) {
551 stamp = rd32(E1000_SYSTIMR) >> 8;
552 shift = IGB_82580_TSYNC_SHIFT;
553 }
554
555 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
556 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
557 return stamp;
558 }
559
560 /**
561 * igb_get_hw_dev - return device
562 * used by hardware layer to print debugging information
563 **/
564 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
565 {
566 struct igb_adapter *adapter = hw->back;
567 return adapter->netdev;
568 }
569
570 /**
571 * igb_init_module - Driver Registration Routine
572 *
573 * igb_init_module is the first routine called when the driver is
574 * loaded. All it does is register with the PCI subsystem.
575 **/
576 static int __init igb_init_module(void)
577 {
578 int ret;
579 printk(KERN_INFO "%s - version %s\n",
580 igb_driver_string, igb_driver_version);
581
582 printk(KERN_INFO "%s\n", igb_copyright);
583
584 #ifdef CONFIG_IGB_DCA
585 dca_register_notify(&dca_notifier);
586 #endif
587 ret = pci_register_driver(&igb_driver);
588 return ret;
589 }
590
591 module_init(igb_init_module);
592
593 /**
594 * igb_exit_module - Driver Exit Cleanup Routine
595 *
596 * igb_exit_module is called just before the driver is removed
597 * from memory.
598 **/
599 static void __exit igb_exit_module(void)
600 {
601 #ifdef CONFIG_IGB_DCA
602 dca_unregister_notify(&dca_notifier);
603 #endif
604 pci_unregister_driver(&igb_driver);
605 }
606
607 module_exit(igb_exit_module);
608
609 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
610 /**
611 * igb_cache_ring_register - Descriptor ring to register mapping
612 * @adapter: board private structure to initialize
613 *
614 * Once we know the feature-set enabled for the device, we'll cache
615 * the register offset the descriptor ring is assigned to.
616 **/
617 static void igb_cache_ring_register(struct igb_adapter *adapter)
618 {
619 int i = 0, j = 0;
620 u32 rbase_offset = adapter->vfs_allocated_count;
621
622 switch (adapter->hw.mac.type) {
623 case e1000_82576:
624 /* The queues are allocated for virtualization such that VF 0
625 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
626 * In order to avoid collision we start at the first free queue
627 * and continue consuming queues in the same sequence
628 */
629 if (adapter->vfs_allocated_count) {
630 for (; i < adapter->rss_queues; i++)
631 adapter->rx_ring[i]->reg_idx = rbase_offset +
632 Q_IDX_82576(i);
633 for (; j < adapter->rss_queues; j++)
634 adapter->tx_ring[j]->reg_idx = rbase_offset +
635 Q_IDX_82576(j);
636 }
637 case e1000_82575:
638 case e1000_82580:
639 case e1000_i350:
640 default:
641 for (; i < adapter->num_rx_queues; i++)
642 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
643 for (; j < adapter->num_tx_queues; j++)
644 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
645 break;
646 }
647 }
648
649 static void igb_free_queues(struct igb_adapter *adapter)
650 {
651 int i;
652
653 for (i = 0; i < adapter->num_tx_queues; i++) {
654 kfree(adapter->tx_ring[i]);
655 adapter->tx_ring[i] = NULL;
656 }
657 for (i = 0; i < adapter->num_rx_queues; i++) {
658 kfree(adapter->rx_ring[i]);
659 adapter->rx_ring[i] = NULL;
660 }
661 adapter->num_rx_queues = 0;
662 adapter->num_tx_queues = 0;
663 }
664
665 /**
666 * igb_alloc_queues - Allocate memory for all rings
667 * @adapter: board private structure to initialize
668 *
669 * We allocate one ring per queue at run-time since we don't know the
670 * number of queues at compile-time.
671 **/
672 static int igb_alloc_queues(struct igb_adapter *adapter)
673 {
674 struct igb_ring *ring;
675 int i;
676
677 for (i = 0; i < adapter->num_tx_queues; i++) {
678 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
679 if (!ring)
680 goto err;
681 ring->count = adapter->tx_ring_count;
682 ring->queue_index = i;
683 ring->dev = &adapter->pdev->dev;
684 ring->netdev = adapter->netdev;
685 /* For 82575, context index must be unique per ring. */
686 if (adapter->hw.mac.type == e1000_82575)
687 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
688 adapter->tx_ring[i] = ring;
689 }
690
691 for (i = 0; i < adapter->num_rx_queues; i++) {
692 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
693 if (!ring)
694 goto err;
695 ring->count = adapter->rx_ring_count;
696 ring->queue_index = i;
697 ring->dev = &adapter->pdev->dev;
698 ring->netdev = adapter->netdev;
699 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
700 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
701 /* set flag indicating ring supports SCTP checksum offload */
702 if (adapter->hw.mac.type >= e1000_82576)
703 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
704 adapter->rx_ring[i] = ring;
705 }
706
707 igb_cache_ring_register(adapter);
708
709 return 0;
710
711 err:
712 igb_free_queues(adapter);
713
714 return -ENOMEM;
715 }
716
717 #define IGB_N0_QUEUE -1
718 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
719 {
720 u32 msixbm = 0;
721 struct igb_adapter *adapter = q_vector->adapter;
722 struct e1000_hw *hw = &adapter->hw;
723 u32 ivar, index;
724 int rx_queue = IGB_N0_QUEUE;
725 int tx_queue = IGB_N0_QUEUE;
726
727 if (q_vector->rx_ring)
728 rx_queue = q_vector->rx_ring->reg_idx;
729 if (q_vector->tx_ring)
730 tx_queue = q_vector->tx_ring->reg_idx;
731
732 switch (hw->mac.type) {
733 case e1000_82575:
734 /* The 82575 assigns vectors using a bitmask, which matches the
735 bitmask for the EICR/EIMS/EIMC registers. To assign one
736 or more queues to a vector, we write the appropriate bits
737 into the MSIXBM register for that vector. */
738 if (rx_queue > IGB_N0_QUEUE)
739 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
740 if (tx_queue > IGB_N0_QUEUE)
741 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
742 if (!adapter->msix_entries && msix_vector == 0)
743 msixbm |= E1000_EIMS_OTHER;
744 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
745 q_vector->eims_value = msixbm;
746 break;
747 case e1000_82576:
748 /* 82576 uses a table-based method for assigning vectors.
749 Each queue has a single entry in the table to which we write
750 a vector number along with a "valid" bit. Sadly, the layout
751 of the table is somewhat counterintuitive. */
752 if (rx_queue > IGB_N0_QUEUE) {
753 index = (rx_queue & 0x7);
754 ivar = array_rd32(E1000_IVAR0, index);
755 if (rx_queue < 8) {
756 /* vector goes into low byte of register */
757 ivar = ivar & 0xFFFFFF00;
758 ivar |= msix_vector | E1000_IVAR_VALID;
759 } else {
760 /* vector goes into third byte of register */
761 ivar = ivar & 0xFF00FFFF;
762 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
763 }
764 array_wr32(E1000_IVAR0, index, ivar);
765 }
766 if (tx_queue > IGB_N0_QUEUE) {
767 index = (tx_queue & 0x7);
768 ivar = array_rd32(E1000_IVAR0, index);
769 if (tx_queue < 8) {
770 /* vector goes into second byte of register */
771 ivar = ivar & 0xFFFF00FF;
772 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
773 } else {
774 /* vector goes into high byte of register */
775 ivar = ivar & 0x00FFFFFF;
776 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
777 }
778 array_wr32(E1000_IVAR0, index, ivar);
779 }
780 q_vector->eims_value = 1 << msix_vector;
781 break;
782 case e1000_82580:
783 case e1000_i350:
784 /* 82580 uses the same table-based approach as 82576 but has fewer
785 entries as a result we carry over for queues greater than 4. */
786 if (rx_queue > IGB_N0_QUEUE) {
787 index = (rx_queue >> 1);
788 ivar = array_rd32(E1000_IVAR0, index);
789 if (rx_queue & 0x1) {
790 /* vector goes into third byte of register */
791 ivar = ivar & 0xFF00FFFF;
792 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
793 } else {
794 /* vector goes into low byte of register */
795 ivar = ivar & 0xFFFFFF00;
796 ivar |= msix_vector | E1000_IVAR_VALID;
797 }
798 array_wr32(E1000_IVAR0, index, ivar);
799 }
800 if (tx_queue > IGB_N0_QUEUE) {
801 index = (tx_queue >> 1);
802 ivar = array_rd32(E1000_IVAR0, index);
803 if (tx_queue & 0x1) {
804 /* vector goes into high byte of register */
805 ivar = ivar & 0x00FFFFFF;
806 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
807 } else {
808 /* vector goes into second byte of register */
809 ivar = ivar & 0xFFFF00FF;
810 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
811 }
812 array_wr32(E1000_IVAR0, index, ivar);
813 }
814 q_vector->eims_value = 1 << msix_vector;
815 break;
816 default:
817 BUG();
818 break;
819 }
820
821 /* add q_vector eims value to global eims_enable_mask */
822 adapter->eims_enable_mask |= q_vector->eims_value;
823
824 /* configure q_vector to set itr on first interrupt */
825 q_vector->set_itr = 1;
826 }
827
828 /**
829 * igb_configure_msix - Configure MSI-X hardware
830 *
831 * igb_configure_msix sets up the hardware to properly
832 * generate MSI-X interrupts.
833 **/
834 static void igb_configure_msix(struct igb_adapter *adapter)
835 {
836 u32 tmp;
837 int i, vector = 0;
838 struct e1000_hw *hw = &adapter->hw;
839
840 adapter->eims_enable_mask = 0;
841
842 /* set vector for other causes, i.e. link changes */
843 switch (hw->mac.type) {
844 case e1000_82575:
845 tmp = rd32(E1000_CTRL_EXT);
846 /* enable MSI-X PBA support*/
847 tmp |= E1000_CTRL_EXT_PBA_CLR;
848
849 /* Auto-Mask interrupts upon ICR read. */
850 tmp |= E1000_CTRL_EXT_EIAME;
851 tmp |= E1000_CTRL_EXT_IRCA;
852
853 wr32(E1000_CTRL_EXT, tmp);
854
855 /* enable msix_other interrupt */
856 array_wr32(E1000_MSIXBM(0), vector++,
857 E1000_EIMS_OTHER);
858 adapter->eims_other = E1000_EIMS_OTHER;
859
860 break;
861
862 case e1000_82576:
863 case e1000_82580:
864 case e1000_i350:
865 /* Turn on MSI-X capability first, or our settings
866 * won't stick. And it will take days to debug. */
867 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
868 E1000_GPIE_PBA | E1000_GPIE_EIAME |
869 E1000_GPIE_NSICR);
870
871 /* enable msix_other interrupt */
872 adapter->eims_other = 1 << vector;
873 tmp = (vector++ | E1000_IVAR_VALID) << 8;
874
875 wr32(E1000_IVAR_MISC, tmp);
876 break;
877 default:
878 /* do nothing, since nothing else supports MSI-X */
879 break;
880 } /* switch (hw->mac.type) */
881
882 adapter->eims_enable_mask |= adapter->eims_other;
883
884 for (i = 0; i < adapter->num_q_vectors; i++)
885 igb_assign_vector(adapter->q_vector[i], vector++);
886
887 wrfl();
888 }
889
890 /**
891 * igb_request_msix - Initialize MSI-X interrupts
892 *
893 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
894 * kernel.
895 **/
896 static int igb_request_msix(struct igb_adapter *adapter)
897 {
898 struct net_device *netdev = adapter->netdev;
899 struct e1000_hw *hw = &adapter->hw;
900 int i, err = 0, vector = 0;
901
902 err = request_irq(adapter->msix_entries[vector].vector,
903 igb_msix_other, 0, netdev->name, adapter);
904 if (err)
905 goto out;
906 vector++;
907
908 for (i = 0; i < adapter->num_q_vectors; i++) {
909 struct igb_q_vector *q_vector = adapter->q_vector[i];
910
911 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
912
913 if (q_vector->rx_ring && q_vector->tx_ring)
914 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
915 q_vector->rx_ring->queue_index);
916 else if (q_vector->tx_ring)
917 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
918 q_vector->tx_ring->queue_index);
919 else if (q_vector->rx_ring)
920 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
921 q_vector->rx_ring->queue_index);
922 else
923 sprintf(q_vector->name, "%s-unused", netdev->name);
924
925 err = request_irq(adapter->msix_entries[vector].vector,
926 igb_msix_ring, 0, q_vector->name,
927 q_vector);
928 if (err)
929 goto out;
930 vector++;
931 }
932
933 igb_configure_msix(adapter);
934 return 0;
935 out:
936 return err;
937 }
938
939 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
940 {
941 if (adapter->msix_entries) {
942 pci_disable_msix(adapter->pdev);
943 kfree(adapter->msix_entries);
944 adapter->msix_entries = NULL;
945 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
946 pci_disable_msi(adapter->pdev);
947 }
948 }
949
950 /**
951 * igb_free_q_vectors - Free memory allocated for interrupt vectors
952 * @adapter: board private structure to initialize
953 *
954 * This function frees the memory allocated to the q_vectors. In addition if
955 * NAPI is enabled it will delete any references to the NAPI struct prior
956 * to freeing the q_vector.
957 **/
958 static void igb_free_q_vectors(struct igb_adapter *adapter)
959 {
960 int v_idx;
961
962 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
963 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
964 adapter->q_vector[v_idx] = NULL;
965 if (!q_vector)
966 continue;
967 netif_napi_del(&q_vector->napi);
968 kfree(q_vector);
969 }
970 adapter->num_q_vectors = 0;
971 }
972
973 /**
974 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
975 *
976 * This function resets the device so that it has 0 rx queues, tx queues, and
977 * MSI-X interrupts allocated.
978 */
979 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
980 {
981 igb_free_queues(adapter);
982 igb_free_q_vectors(adapter);
983 igb_reset_interrupt_capability(adapter);
984 }
985
986 /**
987 * igb_set_interrupt_capability - set MSI or MSI-X if supported
988 *
989 * Attempt to configure interrupts using the best available
990 * capabilities of the hardware and kernel.
991 **/
992 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
993 {
994 int err;
995 int numvecs, i;
996
997 /* Number of supported queues. */
998 adapter->num_rx_queues = adapter->rss_queues;
999 adapter->num_tx_queues = adapter->rss_queues;
1000
1001 /* start with one vector for every rx queue */
1002 numvecs = adapter->num_rx_queues;
1003
1004 /* if tx handler is separate add 1 for every tx queue */
1005 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1006 numvecs += adapter->num_tx_queues;
1007
1008 /* store the number of vectors reserved for queues */
1009 adapter->num_q_vectors = numvecs;
1010
1011 /* add 1 vector for link status interrupts */
1012 numvecs++;
1013 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1014 GFP_KERNEL);
1015 if (!adapter->msix_entries)
1016 goto msi_only;
1017
1018 for (i = 0; i < numvecs; i++)
1019 adapter->msix_entries[i].entry = i;
1020
1021 err = pci_enable_msix(adapter->pdev,
1022 adapter->msix_entries,
1023 numvecs);
1024 if (err == 0)
1025 goto out;
1026
1027 igb_reset_interrupt_capability(adapter);
1028
1029 /* If we can't do MSI-X, try MSI */
1030 msi_only:
1031 #ifdef CONFIG_PCI_IOV
1032 /* disable SR-IOV for non MSI-X configurations */
1033 if (adapter->vf_data) {
1034 struct e1000_hw *hw = &adapter->hw;
1035 /* disable iov and allow time for transactions to clear */
1036 pci_disable_sriov(adapter->pdev);
1037 msleep(500);
1038
1039 kfree(adapter->vf_data);
1040 adapter->vf_data = NULL;
1041 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1042 msleep(100);
1043 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1044 }
1045 #endif
1046 adapter->vfs_allocated_count = 0;
1047 adapter->rss_queues = 1;
1048 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1049 adapter->num_rx_queues = 1;
1050 adapter->num_tx_queues = 1;
1051 adapter->num_q_vectors = 1;
1052 if (!pci_enable_msi(adapter->pdev))
1053 adapter->flags |= IGB_FLAG_HAS_MSI;
1054 out:
1055 /* Notify the stack of the (possibly) reduced Tx Queue count. */
1056 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
1057 }
1058
1059 /**
1060 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1061 * @adapter: board private structure to initialize
1062 *
1063 * We allocate one q_vector per queue interrupt. If allocation fails we
1064 * return -ENOMEM.
1065 **/
1066 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1067 {
1068 struct igb_q_vector *q_vector;
1069 struct e1000_hw *hw = &adapter->hw;
1070 int v_idx;
1071
1072 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1073 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1074 if (!q_vector)
1075 goto err_out;
1076 q_vector->adapter = adapter;
1077 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1078 q_vector->itr_val = IGB_START_ITR;
1079 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1080 adapter->q_vector[v_idx] = q_vector;
1081 }
1082 return 0;
1083
1084 err_out:
1085 igb_free_q_vectors(adapter);
1086 return -ENOMEM;
1087 }
1088
1089 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1090 int ring_idx, int v_idx)
1091 {
1092 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1093
1094 q_vector->rx_ring = adapter->rx_ring[ring_idx];
1095 q_vector->rx_ring->q_vector = q_vector;
1096 q_vector->itr_val = adapter->rx_itr_setting;
1097 if (q_vector->itr_val && q_vector->itr_val <= 3)
1098 q_vector->itr_val = IGB_START_ITR;
1099 }
1100
1101 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1102 int ring_idx, int v_idx)
1103 {
1104 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1105
1106 q_vector->tx_ring = adapter->tx_ring[ring_idx];
1107 q_vector->tx_ring->q_vector = q_vector;
1108 q_vector->itr_val = adapter->tx_itr_setting;
1109 if (q_vector->itr_val && q_vector->itr_val <= 3)
1110 q_vector->itr_val = IGB_START_ITR;
1111 }
1112
1113 /**
1114 * igb_map_ring_to_vector - maps allocated queues to vectors
1115 *
1116 * This function maps the recently allocated queues to vectors.
1117 **/
1118 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1119 {
1120 int i;
1121 int v_idx = 0;
1122
1123 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1124 (adapter->num_q_vectors < adapter->num_tx_queues))
1125 return -ENOMEM;
1126
1127 if (adapter->num_q_vectors >=
1128 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1129 for (i = 0; i < adapter->num_rx_queues; i++)
1130 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1131 for (i = 0; i < adapter->num_tx_queues; i++)
1132 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1133 } else {
1134 for (i = 0; i < adapter->num_rx_queues; i++) {
1135 if (i < adapter->num_tx_queues)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1137 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1138 }
1139 for (; i < adapter->num_tx_queues; i++)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1141 }
1142 return 0;
1143 }
1144
1145 /**
1146 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1147 *
1148 * This function initializes the interrupts and allocates all of the queues.
1149 **/
1150 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1151 {
1152 struct pci_dev *pdev = adapter->pdev;
1153 int err;
1154
1155 igb_set_interrupt_capability(adapter);
1156
1157 err = igb_alloc_q_vectors(adapter);
1158 if (err) {
1159 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1160 goto err_alloc_q_vectors;
1161 }
1162
1163 err = igb_alloc_queues(adapter);
1164 if (err) {
1165 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1166 goto err_alloc_queues;
1167 }
1168
1169 err = igb_map_ring_to_vector(adapter);
1170 if (err) {
1171 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1172 goto err_map_queues;
1173 }
1174
1175
1176 return 0;
1177 err_map_queues:
1178 igb_free_queues(adapter);
1179 err_alloc_queues:
1180 igb_free_q_vectors(adapter);
1181 err_alloc_q_vectors:
1182 igb_reset_interrupt_capability(adapter);
1183 return err;
1184 }
1185
1186 /**
1187 * igb_request_irq - initialize interrupts
1188 *
1189 * Attempts to configure interrupts using the best available
1190 * capabilities of the hardware and kernel.
1191 **/
1192 static int igb_request_irq(struct igb_adapter *adapter)
1193 {
1194 struct net_device *netdev = adapter->netdev;
1195 struct pci_dev *pdev = adapter->pdev;
1196 int err = 0;
1197
1198 if (adapter->msix_entries) {
1199 err = igb_request_msix(adapter);
1200 if (!err)
1201 goto request_done;
1202 /* fall back to MSI */
1203 igb_clear_interrupt_scheme(adapter);
1204 if (!pci_enable_msi(adapter->pdev))
1205 adapter->flags |= IGB_FLAG_HAS_MSI;
1206 igb_free_all_tx_resources(adapter);
1207 igb_free_all_rx_resources(adapter);
1208 adapter->num_tx_queues = 1;
1209 adapter->num_rx_queues = 1;
1210 adapter->num_q_vectors = 1;
1211 err = igb_alloc_q_vectors(adapter);
1212 if (err) {
1213 dev_err(&pdev->dev,
1214 "Unable to allocate memory for vectors\n");
1215 goto request_done;
1216 }
1217 err = igb_alloc_queues(adapter);
1218 if (err) {
1219 dev_err(&pdev->dev,
1220 "Unable to allocate memory for queues\n");
1221 igb_free_q_vectors(adapter);
1222 goto request_done;
1223 }
1224 igb_setup_all_tx_resources(adapter);
1225 igb_setup_all_rx_resources(adapter);
1226 } else {
1227 igb_assign_vector(adapter->q_vector[0], 0);
1228 }
1229
1230 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1231 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
1232 netdev->name, adapter);
1233 if (!err)
1234 goto request_done;
1235
1236 /* fall back to legacy interrupts */
1237 igb_reset_interrupt_capability(adapter);
1238 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1239 }
1240
1241 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
1242 netdev->name, adapter);
1243
1244 if (err)
1245 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1246 err);
1247
1248 request_done:
1249 return err;
1250 }
1251
1252 static void igb_free_irq(struct igb_adapter *adapter)
1253 {
1254 if (adapter->msix_entries) {
1255 int vector = 0, i;
1256
1257 free_irq(adapter->msix_entries[vector++].vector, adapter);
1258
1259 for (i = 0; i < adapter->num_q_vectors; i++) {
1260 struct igb_q_vector *q_vector = adapter->q_vector[i];
1261 free_irq(adapter->msix_entries[vector++].vector,
1262 q_vector);
1263 }
1264 } else {
1265 free_irq(adapter->pdev->irq, adapter);
1266 }
1267 }
1268
1269 /**
1270 * igb_irq_disable - Mask off interrupt generation on the NIC
1271 * @adapter: board private structure
1272 **/
1273 static void igb_irq_disable(struct igb_adapter *adapter)
1274 {
1275 struct e1000_hw *hw = &adapter->hw;
1276
1277 /*
1278 * we need to be careful when disabling interrupts. The VFs are also
1279 * mapped into these registers and so clearing the bits can cause
1280 * issues on the VF drivers so we only need to clear what we set
1281 */
1282 if (adapter->msix_entries) {
1283 u32 regval = rd32(E1000_EIAM);
1284 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1285 wr32(E1000_EIMC, adapter->eims_enable_mask);
1286 regval = rd32(E1000_EIAC);
1287 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1288 }
1289
1290 wr32(E1000_IAM, 0);
1291 wr32(E1000_IMC, ~0);
1292 wrfl();
1293 synchronize_irq(adapter->pdev->irq);
1294 }
1295
1296 /**
1297 * igb_irq_enable - Enable default interrupt generation settings
1298 * @adapter: board private structure
1299 **/
1300 static void igb_irq_enable(struct igb_adapter *adapter)
1301 {
1302 struct e1000_hw *hw = &adapter->hw;
1303
1304 if (adapter->msix_entries) {
1305 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1306 u32 regval = rd32(E1000_EIAC);
1307 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1308 regval = rd32(E1000_EIAM);
1309 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1310 wr32(E1000_EIMS, adapter->eims_enable_mask);
1311 if (adapter->vfs_allocated_count) {
1312 wr32(E1000_MBVFIMR, 0xFF);
1313 ims |= E1000_IMS_VMMB;
1314 }
1315 if (adapter->hw.mac.type == e1000_82580)
1316 ims |= E1000_IMS_DRSTA;
1317
1318 wr32(E1000_IMS, ims);
1319 } else {
1320 wr32(E1000_IMS, IMS_ENABLE_MASK |
1321 E1000_IMS_DRSTA);
1322 wr32(E1000_IAM, IMS_ENABLE_MASK |
1323 E1000_IMS_DRSTA);
1324 }
1325 }
1326
1327 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1328 {
1329 struct e1000_hw *hw = &adapter->hw;
1330 u16 vid = adapter->hw.mng_cookie.vlan_id;
1331 u16 old_vid = adapter->mng_vlan_id;
1332
1333 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1334 /* add VID to filter table */
1335 igb_vfta_set(hw, vid, true);
1336 adapter->mng_vlan_id = vid;
1337 } else {
1338 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1339 }
1340
1341 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1342 (vid != old_vid) &&
1343 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1344 /* remove VID from filter table */
1345 igb_vfta_set(hw, old_vid, false);
1346 }
1347 }
1348
1349 /**
1350 * igb_release_hw_control - release control of the h/w to f/w
1351 * @adapter: address of board private structure
1352 *
1353 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1354 * For ASF and Pass Through versions of f/w this means that the
1355 * driver is no longer loaded.
1356 *
1357 **/
1358 static void igb_release_hw_control(struct igb_adapter *adapter)
1359 {
1360 struct e1000_hw *hw = &adapter->hw;
1361 u32 ctrl_ext;
1362
1363 /* Let firmware take over control of h/w */
1364 ctrl_ext = rd32(E1000_CTRL_EXT);
1365 wr32(E1000_CTRL_EXT,
1366 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1367 }
1368
1369 /**
1370 * igb_get_hw_control - get control of the h/w from f/w
1371 * @adapter: address of board private structure
1372 *
1373 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1374 * For ASF and Pass Through versions of f/w this means that
1375 * the driver is loaded.
1376 *
1377 **/
1378 static void igb_get_hw_control(struct igb_adapter *adapter)
1379 {
1380 struct e1000_hw *hw = &adapter->hw;
1381 u32 ctrl_ext;
1382
1383 /* Let firmware know the driver has taken over */
1384 ctrl_ext = rd32(E1000_CTRL_EXT);
1385 wr32(E1000_CTRL_EXT,
1386 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1387 }
1388
1389 /**
1390 * igb_configure - configure the hardware for RX and TX
1391 * @adapter: private board structure
1392 **/
1393 static void igb_configure(struct igb_adapter *adapter)
1394 {
1395 struct net_device *netdev = adapter->netdev;
1396 int i;
1397
1398 igb_get_hw_control(adapter);
1399 igb_set_rx_mode(netdev);
1400
1401 igb_restore_vlan(adapter);
1402
1403 igb_setup_tctl(adapter);
1404 igb_setup_mrqc(adapter);
1405 igb_setup_rctl(adapter);
1406
1407 igb_configure_tx(adapter);
1408 igb_configure_rx(adapter);
1409
1410 igb_rx_fifo_flush_82575(&adapter->hw);
1411
1412 /* call igb_desc_unused which always leaves
1413 * at least 1 descriptor unused to make sure
1414 * next_to_use != next_to_clean */
1415 for (i = 0; i < adapter->num_rx_queues; i++) {
1416 struct igb_ring *ring = adapter->rx_ring[i];
1417 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1418 }
1419 }
1420
1421 /**
1422 * igb_power_up_link - Power up the phy/serdes link
1423 * @adapter: address of board private structure
1424 **/
1425 void igb_power_up_link(struct igb_adapter *adapter)
1426 {
1427 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1428 igb_power_up_phy_copper(&adapter->hw);
1429 else
1430 igb_power_up_serdes_link_82575(&adapter->hw);
1431 }
1432
1433 /**
1434 * igb_power_down_link - Power down the phy/serdes link
1435 * @adapter: address of board private structure
1436 */
1437 static void igb_power_down_link(struct igb_adapter *adapter)
1438 {
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_down_phy_copper_82575(&adapter->hw);
1441 else
1442 igb_shutdown_serdes_link_82575(&adapter->hw);
1443 }
1444
1445 /**
1446 * igb_up - Open the interface and prepare it to handle traffic
1447 * @adapter: board private structure
1448 **/
1449 int igb_up(struct igb_adapter *adapter)
1450 {
1451 struct e1000_hw *hw = &adapter->hw;
1452 int i;
1453
1454 /* hardware has been reset, we need to reload some things */
1455 igb_configure(adapter);
1456
1457 clear_bit(__IGB_DOWN, &adapter->state);
1458
1459 for (i = 0; i < adapter->num_q_vectors; i++) {
1460 struct igb_q_vector *q_vector = adapter->q_vector[i];
1461 napi_enable(&q_vector->napi);
1462 }
1463 if (adapter->msix_entries)
1464 igb_configure_msix(adapter);
1465 else
1466 igb_assign_vector(adapter->q_vector[0], 0);
1467
1468 /* Clear any pending interrupts. */
1469 rd32(E1000_ICR);
1470 igb_irq_enable(adapter);
1471
1472 /* notify VFs that reset has been completed */
1473 if (adapter->vfs_allocated_count) {
1474 u32 reg_data = rd32(E1000_CTRL_EXT);
1475 reg_data |= E1000_CTRL_EXT_PFRSTD;
1476 wr32(E1000_CTRL_EXT, reg_data);
1477 }
1478
1479 netif_tx_start_all_queues(adapter->netdev);
1480
1481 /* start the watchdog. */
1482 hw->mac.get_link_status = 1;
1483 schedule_work(&adapter->watchdog_task);
1484
1485 return 0;
1486 }
1487
1488 void igb_down(struct igb_adapter *adapter)
1489 {
1490 struct net_device *netdev = adapter->netdev;
1491 struct e1000_hw *hw = &adapter->hw;
1492 u32 tctl, rctl;
1493 int i;
1494
1495 /* signal that we're down so the interrupt handler does not
1496 * reschedule our watchdog timer */
1497 set_bit(__IGB_DOWN, &adapter->state);
1498
1499 /* disable receives in the hardware */
1500 rctl = rd32(E1000_RCTL);
1501 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1502 /* flush and sleep below */
1503
1504 netif_tx_stop_all_queues(netdev);
1505
1506 /* disable transmits in the hardware */
1507 tctl = rd32(E1000_TCTL);
1508 tctl &= ~E1000_TCTL_EN;
1509 wr32(E1000_TCTL, tctl);
1510 /* flush both disables and wait for them to finish */
1511 wrfl();
1512 msleep(10);
1513
1514 for (i = 0; i < adapter->num_q_vectors; i++) {
1515 struct igb_q_vector *q_vector = adapter->q_vector[i];
1516 napi_disable(&q_vector->napi);
1517 }
1518
1519 igb_irq_disable(adapter);
1520
1521 del_timer_sync(&adapter->watchdog_timer);
1522 del_timer_sync(&adapter->phy_info_timer);
1523
1524 netif_carrier_off(netdev);
1525
1526 /* record the stats before reset*/
1527 igb_update_stats(adapter);
1528
1529 adapter->link_speed = 0;
1530 adapter->link_duplex = 0;
1531
1532 if (!pci_channel_offline(adapter->pdev))
1533 igb_reset(adapter);
1534 igb_clean_all_tx_rings(adapter);
1535 igb_clean_all_rx_rings(adapter);
1536 #ifdef CONFIG_IGB_DCA
1537
1538 /* since we reset the hardware DCA settings were cleared */
1539 igb_setup_dca(adapter);
1540 #endif
1541 }
1542
1543 void igb_reinit_locked(struct igb_adapter *adapter)
1544 {
1545 WARN_ON(in_interrupt());
1546 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1547 msleep(1);
1548 igb_down(adapter);
1549 igb_up(adapter);
1550 clear_bit(__IGB_RESETTING, &adapter->state);
1551 }
1552
1553 void igb_reset(struct igb_adapter *adapter)
1554 {
1555 struct pci_dev *pdev = adapter->pdev;
1556 struct e1000_hw *hw = &adapter->hw;
1557 struct e1000_mac_info *mac = &hw->mac;
1558 struct e1000_fc_info *fc = &hw->fc;
1559 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1560 u16 hwm;
1561
1562 /* Repartition Pba for greater than 9k mtu
1563 * To take effect CTRL.RST is required.
1564 */
1565 switch (mac->type) {
1566 case e1000_i350:
1567 case e1000_82580:
1568 pba = rd32(E1000_RXPBS);
1569 pba = igb_rxpbs_adjust_82580(pba);
1570 break;
1571 case e1000_82576:
1572 pba = rd32(E1000_RXPBS);
1573 pba &= E1000_RXPBS_SIZE_MASK_82576;
1574 break;
1575 case e1000_82575:
1576 default:
1577 pba = E1000_PBA_34K;
1578 break;
1579 }
1580
1581 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1582 (mac->type < e1000_82576)) {
1583 /* adjust PBA for jumbo frames */
1584 wr32(E1000_PBA, pba);
1585
1586 /* To maintain wire speed transmits, the Tx FIFO should be
1587 * large enough to accommodate two full transmit packets,
1588 * rounded up to the next 1KB and expressed in KB. Likewise,
1589 * the Rx FIFO should be large enough to accommodate at least
1590 * one full receive packet and is similarly rounded up and
1591 * expressed in KB. */
1592 pba = rd32(E1000_PBA);
1593 /* upper 16 bits has Tx packet buffer allocation size in KB */
1594 tx_space = pba >> 16;
1595 /* lower 16 bits has Rx packet buffer allocation size in KB */
1596 pba &= 0xffff;
1597 /* the tx fifo also stores 16 bytes of information about the tx
1598 * but don't include ethernet FCS because hardware appends it */
1599 min_tx_space = (adapter->max_frame_size +
1600 sizeof(union e1000_adv_tx_desc) -
1601 ETH_FCS_LEN) * 2;
1602 min_tx_space = ALIGN(min_tx_space, 1024);
1603 min_tx_space >>= 10;
1604 /* software strips receive CRC, so leave room for it */
1605 min_rx_space = adapter->max_frame_size;
1606 min_rx_space = ALIGN(min_rx_space, 1024);
1607 min_rx_space >>= 10;
1608
1609 /* If current Tx allocation is less than the min Tx FIFO size,
1610 * and the min Tx FIFO size is less than the current Rx FIFO
1611 * allocation, take space away from current Rx allocation */
1612 if (tx_space < min_tx_space &&
1613 ((min_tx_space - tx_space) < pba)) {
1614 pba = pba - (min_tx_space - tx_space);
1615
1616 /* if short on rx space, rx wins and must trump tx
1617 * adjustment */
1618 if (pba < min_rx_space)
1619 pba = min_rx_space;
1620 }
1621 wr32(E1000_PBA, pba);
1622 }
1623
1624 /* flow control settings */
1625 /* The high water mark must be low enough to fit one full frame
1626 * (or the size used for early receive) above it in the Rx FIFO.
1627 * Set it to the lower of:
1628 * - 90% of the Rx FIFO size, or
1629 * - the full Rx FIFO size minus one full frame */
1630 hwm = min(((pba << 10) * 9 / 10),
1631 ((pba << 10) - 2 * adapter->max_frame_size));
1632
1633 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1634 fc->low_water = fc->high_water - 16;
1635 fc->pause_time = 0xFFFF;
1636 fc->send_xon = 1;
1637 fc->current_mode = fc->requested_mode;
1638
1639 /* disable receive for all VFs and wait one second */
1640 if (adapter->vfs_allocated_count) {
1641 int i;
1642 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1643 adapter->vf_data[i].flags = 0;
1644
1645 /* ping all the active vfs to let them know we are going down */
1646 igb_ping_all_vfs(adapter);
1647
1648 /* disable transmits and receives */
1649 wr32(E1000_VFRE, 0);
1650 wr32(E1000_VFTE, 0);
1651 }
1652
1653 /* Allow time for pending master requests to run */
1654 hw->mac.ops.reset_hw(hw);
1655 wr32(E1000_WUC, 0);
1656
1657 if (hw->mac.ops.init_hw(hw))
1658 dev_err(&pdev->dev, "Hardware Error\n");
1659
1660 if (hw->mac.type == e1000_82580) {
1661 u32 reg = rd32(E1000_PCIEMISC);
1662 wr32(E1000_PCIEMISC,
1663 reg & ~E1000_PCIEMISC_LX_DECISION);
1664 }
1665 if (!netif_running(adapter->netdev))
1666 igb_power_down_link(adapter);
1667
1668 igb_update_mng_vlan(adapter);
1669
1670 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1671 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1672
1673 igb_get_phy_info(hw);
1674 }
1675
1676 static const struct net_device_ops igb_netdev_ops = {
1677 .ndo_open = igb_open,
1678 .ndo_stop = igb_close,
1679 .ndo_start_xmit = igb_xmit_frame_adv,
1680 .ndo_get_stats = igb_get_stats,
1681 .ndo_set_rx_mode = igb_set_rx_mode,
1682 .ndo_set_multicast_list = igb_set_rx_mode,
1683 .ndo_set_mac_address = igb_set_mac,
1684 .ndo_change_mtu = igb_change_mtu,
1685 .ndo_do_ioctl = igb_ioctl,
1686 .ndo_tx_timeout = igb_tx_timeout,
1687 .ndo_validate_addr = eth_validate_addr,
1688 .ndo_vlan_rx_register = igb_vlan_rx_register,
1689 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1690 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1691 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1692 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1693 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1694 .ndo_get_vf_config = igb_ndo_get_vf_config,
1695 #ifdef CONFIG_NET_POLL_CONTROLLER
1696 .ndo_poll_controller = igb_netpoll,
1697 #endif
1698 };
1699
1700 /**
1701 * igb_probe - Device Initialization Routine
1702 * @pdev: PCI device information struct
1703 * @ent: entry in igb_pci_tbl
1704 *
1705 * Returns 0 on success, negative on failure
1706 *
1707 * igb_probe initializes an adapter identified by a pci_dev structure.
1708 * The OS initialization, configuring of the adapter private structure,
1709 * and a hardware reset occur.
1710 **/
1711 static int __devinit igb_probe(struct pci_dev *pdev,
1712 const struct pci_device_id *ent)
1713 {
1714 struct net_device *netdev;
1715 struct igb_adapter *adapter;
1716 struct e1000_hw *hw;
1717 u16 eeprom_data = 0;
1718 static int global_quad_port_a; /* global quad port a indication */
1719 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1720 unsigned long mmio_start, mmio_len;
1721 int err, pci_using_dac;
1722 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1723 u32 part_num;
1724
1725 /* Catch broken hardware that put the wrong VF device ID in
1726 * the PCIe SR-IOV capability.
1727 */
1728 if (pdev->is_virtfn) {
1729 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1730 pci_name(pdev), pdev->vendor, pdev->device);
1731 return -EINVAL;
1732 }
1733
1734 err = pci_enable_device_mem(pdev);
1735 if (err)
1736 return err;
1737
1738 pci_using_dac = 0;
1739 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1740 if (!err) {
1741 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1742 if (!err)
1743 pci_using_dac = 1;
1744 } else {
1745 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1746 if (err) {
1747 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1748 if (err) {
1749 dev_err(&pdev->dev, "No usable DMA "
1750 "configuration, aborting\n");
1751 goto err_dma;
1752 }
1753 }
1754 }
1755
1756 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1757 IORESOURCE_MEM),
1758 igb_driver_name);
1759 if (err)
1760 goto err_pci_reg;
1761
1762 pci_enable_pcie_error_reporting(pdev);
1763
1764 pci_set_master(pdev);
1765 pci_save_state(pdev);
1766
1767 err = -ENOMEM;
1768 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1769 IGB_ABS_MAX_TX_QUEUES);
1770 if (!netdev)
1771 goto err_alloc_etherdev;
1772
1773 SET_NETDEV_DEV(netdev, &pdev->dev);
1774
1775 pci_set_drvdata(pdev, netdev);
1776 adapter = netdev_priv(netdev);
1777 adapter->netdev = netdev;
1778 adapter->pdev = pdev;
1779 hw = &adapter->hw;
1780 hw->back = adapter;
1781 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1782
1783 mmio_start = pci_resource_start(pdev, 0);
1784 mmio_len = pci_resource_len(pdev, 0);
1785
1786 err = -EIO;
1787 hw->hw_addr = ioremap(mmio_start, mmio_len);
1788 if (!hw->hw_addr)
1789 goto err_ioremap;
1790
1791 netdev->netdev_ops = &igb_netdev_ops;
1792 igb_set_ethtool_ops(netdev);
1793 netdev->watchdog_timeo = 5 * HZ;
1794
1795 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1796
1797 netdev->mem_start = mmio_start;
1798 netdev->mem_end = mmio_start + mmio_len;
1799
1800 /* PCI config space info */
1801 hw->vendor_id = pdev->vendor;
1802 hw->device_id = pdev->device;
1803 hw->revision_id = pdev->revision;
1804 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1805 hw->subsystem_device_id = pdev->subsystem_device;
1806
1807 /* Copy the default MAC, PHY and NVM function pointers */
1808 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1809 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1810 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1811 /* Initialize skew-specific constants */
1812 err = ei->get_invariants(hw);
1813 if (err)
1814 goto err_sw_init;
1815
1816 /* setup the private structure */
1817 err = igb_sw_init(adapter);
1818 if (err)
1819 goto err_sw_init;
1820
1821 igb_get_bus_info_pcie(hw);
1822
1823 hw->phy.autoneg_wait_to_complete = false;
1824
1825 /* Copper options */
1826 if (hw->phy.media_type == e1000_media_type_copper) {
1827 hw->phy.mdix = AUTO_ALL_MODES;
1828 hw->phy.disable_polarity_correction = false;
1829 hw->phy.ms_type = e1000_ms_hw_default;
1830 }
1831
1832 if (igb_check_reset_block(hw))
1833 dev_info(&pdev->dev,
1834 "PHY reset is blocked due to SOL/IDER session.\n");
1835
1836 netdev->features = NETIF_F_SG |
1837 NETIF_F_IP_CSUM |
1838 NETIF_F_HW_VLAN_TX |
1839 NETIF_F_HW_VLAN_RX |
1840 NETIF_F_HW_VLAN_FILTER;
1841
1842 netdev->features |= NETIF_F_IPV6_CSUM;
1843 netdev->features |= NETIF_F_TSO;
1844 netdev->features |= NETIF_F_TSO6;
1845 netdev->features |= NETIF_F_GRO;
1846
1847 netdev->vlan_features |= NETIF_F_TSO;
1848 netdev->vlan_features |= NETIF_F_TSO6;
1849 netdev->vlan_features |= NETIF_F_IP_CSUM;
1850 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1851 netdev->vlan_features |= NETIF_F_SG;
1852
1853 if (pci_using_dac)
1854 netdev->features |= NETIF_F_HIGHDMA;
1855
1856 if (hw->mac.type >= e1000_82576)
1857 netdev->features |= NETIF_F_SCTP_CSUM;
1858
1859 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1860
1861 /* before reading the NVM, reset the controller to put the device in a
1862 * known good starting state */
1863 hw->mac.ops.reset_hw(hw);
1864
1865 /* make sure the NVM is good */
1866 if (igb_validate_nvm_checksum(hw) < 0) {
1867 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1868 err = -EIO;
1869 goto err_eeprom;
1870 }
1871
1872 /* copy the MAC address out of the NVM */
1873 if (hw->mac.ops.read_mac_addr(hw))
1874 dev_err(&pdev->dev, "NVM Read Error\n");
1875
1876 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1877 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1878
1879 if (!is_valid_ether_addr(netdev->perm_addr)) {
1880 dev_err(&pdev->dev, "Invalid MAC Address\n");
1881 err = -EIO;
1882 goto err_eeprom;
1883 }
1884
1885 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1886 (unsigned long) adapter);
1887 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1888 (unsigned long) adapter);
1889
1890 INIT_WORK(&adapter->reset_task, igb_reset_task);
1891 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1892
1893 /* Initialize link properties that are user-changeable */
1894 adapter->fc_autoneg = true;
1895 hw->mac.autoneg = true;
1896 hw->phy.autoneg_advertised = 0x2f;
1897
1898 hw->fc.requested_mode = e1000_fc_default;
1899 hw->fc.current_mode = e1000_fc_default;
1900
1901 igb_validate_mdi_setting(hw);
1902
1903 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1904 * enable the ACPI Magic Packet filter
1905 */
1906
1907 if (hw->bus.func == 0)
1908 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1909 else if (hw->mac.type == e1000_82580)
1910 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1911 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1912 &eeprom_data);
1913 else if (hw->bus.func == 1)
1914 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1915
1916 if (eeprom_data & eeprom_apme_mask)
1917 adapter->eeprom_wol |= E1000_WUFC_MAG;
1918
1919 /* now that we have the eeprom settings, apply the special cases where
1920 * the eeprom may be wrong or the board simply won't support wake on
1921 * lan on a particular port */
1922 switch (pdev->device) {
1923 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1924 adapter->eeprom_wol = 0;
1925 break;
1926 case E1000_DEV_ID_82575EB_FIBER_SERDES:
1927 case E1000_DEV_ID_82576_FIBER:
1928 case E1000_DEV_ID_82576_SERDES:
1929 /* Wake events only supported on port A for dual fiber
1930 * regardless of eeprom setting */
1931 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1932 adapter->eeprom_wol = 0;
1933 break;
1934 case E1000_DEV_ID_82576_QUAD_COPPER:
1935 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
1936 /* if quad port adapter, disable WoL on all but port A */
1937 if (global_quad_port_a != 0)
1938 adapter->eeprom_wol = 0;
1939 else
1940 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1941 /* Reset for multiple quad port adapters */
1942 if (++global_quad_port_a == 4)
1943 global_quad_port_a = 0;
1944 break;
1945 }
1946
1947 /* initialize the wol settings based on the eeprom settings */
1948 adapter->wol = adapter->eeprom_wol;
1949 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1950
1951 /* reset the hardware with the new settings */
1952 igb_reset(adapter);
1953
1954 /* let the f/w know that the h/w is now under the control of the
1955 * driver. */
1956 igb_get_hw_control(adapter);
1957
1958 strcpy(netdev->name, "eth%d");
1959 err = register_netdev(netdev);
1960 if (err)
1961 goto err_register;
1962
1963 /* carrier off reporting is important to ethtool even BEFORE open */
1964 netif_carrier_off(netdev);
1965
1966 #ifdef CONFIG_IGB_DCA
1967 if (dca_add_requester(&pdev->dev) == 0) {
1968 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1969 dev_info(&pdev->dev, "DCA enabled\n");
1970 igb_setup_dca(adapter);
1971 }
1972
1973 #endif
1974 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1975 /* print bus type/speed/width info */
1976 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1977 netdev->name,
1978 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1979 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
1980 "unknown"),
1981 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1982 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1983 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1984 "unknown"),
1985 netdev->dev_addr);
1986
1987 igb_read_part_num(hw, &part_num);
1988 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1989 (part_num >> 8), (part_num & 0xff));
1990
1991 dev_info(&pdev->dev,
1992 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1993 adapter->msix_entries ? "MSI-X" :
1994 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
1995 adapter->num_rx_queues, adapter->num_tx_queues);
1996
1997 return 0;
1998
1999 err_register:
2000 igb_release_hw_control(adapter);
2001 err_eeprom:
2002 if (!igb_check_reset_block(hw))
2003 igb_reset_phy(hw);
2004
2005 if (hw->flash_address)
2006 iounmap(hw->flash_address);
2007 err_sw_init:
2008 igb_clear_interrupt_scheme(adapter);
2009 iounmap(hw->hw_addr);
2010 err_ioremap:
2011 free_netdev(netdev);
2012 err_alloc_etherdev:
2013 pci_release_selected_regions(pdev,
2014 pci_select_bars(pdev, IORESOURCE_MEM));
2015 err_pci_reg:
2016 err_dma:
2017 pci_disable_device(pdev);
2018 return err;
2019 }
2020
2021 /**
2022 * igb_remove - Device Removal Routine
2023 * @pdev: PCI device information struct
2024 *
2025 * igb_remove is called by the PCI subsystem to alert the driver
2026 * that it should release a PCI device. The could be caused by a
2027 * Hot-Plug event, or because the driver is going to be removed from
2028 * memory.
2029 **/
2030 static void __devexit igb_remove(struct pci_dev *pdev)
2031 {
2032 struct net_device *netdev = pci_get_drvdata(pdev);
2033 struct igb_adapter *adapter = netdev_priv(netdev);
2034 struct e1000_hw *hw = &adapter->hw;
2035
2036 /* flush_scheduled work may reschedule our watchdog task, so
2037 * explicitly disable watchdog tasks from being rescheduled */
2038 set_bit(__IGB_DOWN, &adapter->state);
2039 del_timer_sync(&adapter->watchdog_timer);
2040 del_timer_sync(&adapter->phy_info_timer);
2041
2042 flush_scheduled_work();
2043
2044 #ifdef CONFIG_IGB_DCA
2045 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2046 dev_info(&pdev->dev, "DCA disabled\n");
2047 dca_remove_requester(&pdev->dev);
2048 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2049 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2050 }
2051 #endif
2052
2053 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2054 * would have already happened in close and is redundant. */
2055 igb_release_hw_control(adapter);
2056
2057 unregister_netdev(netdev);
2058
2059 igb_clear_interrupt_scheme(adapter);
2060
2061 #ifdef CONFIG_PCI_IOV
2062 /* reclaim resources allocated to VFs */
2063 if (adapter->vf_data) {
2064 /* disable iov and allow time for transactions to clear */
2065 pci_disable_sriov(pdev);
2066 msleep(500);
2067
2068 kfree(adapter->vf_data);
2069 adapter->vf_data = NULL;
2070 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2071 msleep(100);
2072 dev_info(&pdev->dev, "IOV Disabled\n");
2073 }
2074 #endif
2075
2076 iounmap(hw->hw_addr);
2077 if (hw->flash_address)
2078 iounmap(hw->flash_address);
2079 pci_release_selected_regions(pdev,
2080 pci_select_bars(pdev, IORESOURCE_MEM));
2081
2082 free_netdev(netdev);
2083
2084 pci_disable_pcie_error_reporting(pdev);
2085
2086 pci_disable_device(pdev);
2087 }
2088
2089 /**
2090 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2091 * @adapter: board private structure to initialize
2092 *
2093 * This function initializes the vf specific data storage and then attempts to
2094 * allocate the VFs. The reason for ordering it this way is because it is much
2095 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2096 * the memory for the VFs.
2097 **/
2098 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2099 {
2100 #ifdef CONFIG_PCI_IOV
2101 struct pci_dev *pdev = adapter->pdev;
2102
2103 if (adapter->vfs_allocated_count > 7)
2104 adapter->vfs_allocated_count = 7;
2105
2106 if (adapter->vfs_allocated_count) {
2107 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2108 sizeof(struct vf_data_storage),
2109 GFP_KERNEL);
2110 /* if allocation failed then we do not support SR-IOV */
2111 if (!adapter->vf_data) {
2112 adapter->vfs_allocated_count = 0;
2113 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2114 "Data Storage\n");
2115 }
2116 }
2117
2118 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2119 kfree(adapter->vf_data);
2120 adapter->vf_data = NULL;
2121 #endif /* CONFIG_PCI_IOV */
2122 adapter->vfs_allocated_count = 0;
2123 #ifdef CONFIG_PCI_IOV
2124 } else {
2125 unsigned char mac_addr[ETH_ALEN];
2126 int i;
2127 dev_info(&pdev->dev, "%d vfs allocated\n",
2128 adapter->vfs_allocated_count);
2129 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2130 random_ether_addr(mac_addr);
2131 igb_set_vf_mac(adapter, i, mac_addr);
2132 }
2133 }
2134 #endif /* CONFIG_PCI_IOV */
2135 }
2136
2137
2138 /**
2139 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2140 * @adapter: board private structure to initialize
2141 *
2142 * igb_init_hw_timer initializes the function pointer and values for the hw
2143 * timer found in hardware.
2144 **/
2145 static void igb_init_hw_timer(struct igb_adapter *adapter)
2146 {
2147 struct e1000_hw *hw = &adapter->hw;
2148
2149 switch (hw->mac.type) {
2150 case e1000_i350:
2151 case e1000_82580:
2152 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2153 adapter->cycles.read = igb_read_clock;
2154 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2155 adapter->cycles.mult = 1;
2156 /*
2157 * The 82580 timesync updates the system timer every 8ns by 8ns
2158 * and the value cannot be shifted. Instead we need to shift
2159 * the registers to generate a 64bit timer value. As a result
2160 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2161 * 24 in order to generate a larger value for synchronization.
2162 */
2163 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2164 /* disable system timer temporarily by setting bit 31 */
2165 wr32(E1000_TSAUXC, 0x80000000);
2166 wrfl();
2167
2168 /* Set registers so that rollover occurs soon to test this. */
2169 wr32(E1000_SYSTIMR, 0x00000000);
2170 wr32(E1000_SYSTIML, 0x80000000);
2171 wr32(E1000_SYSTIMH, 0x000000FF);
2172 wrfl();
2173
2174 /* enable system timer by clearing bit 31 */
2175 wr32(E1000_TSAUXC, 0x0);
2176 wrfl();
2177
2178 timecounter_init(&adapter->clock,
2179 &adapter->cycles,
2180 ktime_to_ns(ktime_get_real()));
2181 /*
2182 * Synchronize our NIC clock against system wall clock. NIC
2183 * time stamp reading requires ~3us per sample, each sample
2184 * was pretty stable even under load => only require 10
2185 * samples for each offset comparison.
2186 */
2187 memset(&adapter->compare, 0, sizeof(adapter->compare));
2188 adapter->compare.source = &adapter->clock;
2189 adapter->compare.target = ktime_get_real;
2190 adapter->compare.num_samples = 10;
2191 timecompare_update(&adapter->compare, 0);
2192 break;
2193 case e1000_82576:
2194 /*
2195 * Initialize hardware timer: we keep it running just in case
2196 * that some program needs it later on.
2197 */
2198 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2199 adapter->cycles.read = igb_read_clock;
2200 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2201 adapter->cycles.mult = 1;
2202 /**
2203 * Scale the NIC clock cycle by a large factor so that
2204 * relatively small clock corrections can be added or
2205 * substracted at each clock tick. The drawbacks of a large
2206 * factor are a) that the clock register overflows more quickly
2207 * (not such a big deal) and b) that the increment per tick has
2208 * to fit into 24 bits. As a result we need to use a shift of
2209 * 19 so we can fit a value of 16 into the TIMINCA register.
2210 */
2211 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2212 wr32(E1000_TIMINCA,
2213 (1 << E1000_TIMINCA_16NS_SHIFT) |
2214 (16 << IGB_82576_TSYNC_SHIFT));
2215
2216 /* Set registers so that rollover occurs soon to test this. */
2217 wr32(E1000_SYSTIML, 0x00000000);
2218 wr32(E1000_SYSTIMH, 0xFF800000);
2219 wrfl();
2220
2221 timecounter_init(&adapter->clock,
2222 &adapter->cycles,
2223 ktime_to_ns(ktime_get_real()));
2224 /*
2225 * Synchronize our NIC clock against system wall clock. NIC
2226 * time stamp reading requires ~3us per sample, each sample
2227 * was pretty stable even under load => only require 10
2228 * samples for each offset comparison.
2229 */
2230 memset(&adapter->compare, 0, sizeof(adapter->compare));
2231 adapter->compare.source = &adapter->clock;
2232 adapter->compare.target = ktime_get_real;
2233 adapter->compare.num_samples = 10;
2234 timecompare_update(&adapter->compare, 0);
2235 break;
2236 case e1000_82575:
2237 /* 82575 does not support timesync */
2238 default:
2239 break;
2240 }
2241
2242 }
2243
2244 /**
2245 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2246 * @adapter: board private structure to initialize
2247 *
2248 * igb_sw_init initializes the Adapter private data structure.
2249 * Fields are initialized based on PCI device information and
2250 * OS network device settings (MTU size).
2251 **/
2252 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2253 {
2254 struct e1000_hw *hw = &adapter->hw;
2255 struct net_device *netdev = adapter->netdev;
2256 struct pci_dev *pdev = adapter->pdev;
2257
2258 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2259
2260 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2261 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2262 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2263 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2264
2265 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2266 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2267
2268 #ifdef CONFIG_PCI_IOV
2269 if (hw->mac.type == e1000_82576)
2270 adapter->vfs_allocated_count = max_vfs;
2271
2272 #endif /* CONFIG_PCI_IOV */
2273 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2274
2275 /*
2276 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2277 * then we should combine the queues into a queue pair in order to
2278 * conserve interrupts due to limited supply
2279 */
2280 if ((adapter->rss_queues > 4) ||
2281 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2282 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2283
2284 /* This call may decrease the number of queues */
2285 if (igb_init_interrupt_scheme(adapter)) {
2286 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2287 return -ENOMEM;
2288 }
2289
2290 igb_init_hw_timer(adapter);
2291 igb_probe_vfs(adapter);
2292
2293 /* Explicitly disable IRQ since the NIC can be in any state. */
2294 igb_irq_disable(adapter);
2295
2296 set_bit(__IGB_DOWN, &adapter->state);
2297 return 0;
2298 }
2299
2300 /**
2301 * igb_open - Called when a network interface is made active
2302 * @netdev: network interface device structure
2303 *
2304 * Returns 0 on success, negative value on failure
2305 *
2306 * The open entry point is called when a network interface is made
2307 * active by the system (IFF_UP). At this point all resources needed
2308 * for transmit and receive operations are allocated, the interrupt
2309 * handler is registered with the OS, the watchdog timer is started,
2310 * and the stack is notified that the interface is ready.
2311 **/
2312 static int igb_open(struct net_device *netdev)
2313 {
2314 struct igb_adapter *adapter = netdev_priv(netdev);
2315 struct e1000_hw *hw = &adapter->hw;
2316 int err;
2317 int i;
2318
2319 /* disallow open during test */
2320 if (test_bit(__IGB_TESTING, &adapter->state))
2321 return -EBUSY;
2322
2323 netif_carrier_off(netdev);
2324
2325 /* allocate transmit descriptors */
2326 err = igb_setup_all_tx_resources(adapter);
2327 if (err)
2328 goto err_setup_tx;
2329
2330 /* allocate receive descriptors */
2331 err = igb_setup_all_rx_resources(adapter);
2332 if (err)
2333 goto err_setup_rx;
2334
2335 igb_power_up_link(adapter);
2336
2337 /* before we allocate an interrupt, we must be ready to handle it.
2338 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2339 * as soon as we call pci_request_irq, so we have to setup our
2340 * clean_rx handler before we do so. */
2341 igb_configure(adapter);
2342
2343 err = igb_request_irq(adapter);
2344 if (err)
2345 goto err_req_irq;
2346
2347 /* From here on the code is the same as igb_up() */
2348 clear_bit(__IGB_DOWN, &adapter->state);
2349
2350 for (i = 0; i < adapter->num_q_vectors; i++) {
2351 struct igb_q_vector *q_vector = adapter->q_vector[i];
2352 napi_enable(&q_vector->napi);
2353 }
2354
2355 /* Clear any pending interrupts. */
2356 rd32(E1000_ICR);
2357
2358 igb_irq_enable(adapter);
2359
2360 /* notify VFs that reset has been completed */
2361 if (adapter->vfs_allocated_count) {
2362 u32 reg_data = rd32(E1000_CTRL_EXT);
2363 reg_data |= E1000_CTRL_EXT_PFRSTD;
2364 wr32(E1000_CTRL_EXT, reg_data);
2365 }
2366
2367 netif_tx_start_all_queues(netdev);
2368
2369 /* start the watchdog. */
2370 hw->mac.get_link_status = 1;
2371 schedule_work(&adapter->watchdog_task);
2372
2373 return 0;
2374
2375 err_req_irq:
2376 igb_release_hw_control(adapter);
2377 igb_power_down_link(adapter);
2378 igb_free_all_rx_resources(adapter);
2379 err_setup_rx:
2380 igb_free_all_tx_resources(adapter);
2381 err_setup_tx:
2382 igb_reset(adapter);
2383
2384 return err;
2385 }
2386
2387 /**
2388 * igb_close - Disables a network interface
2389 * @netdev: network interface device structure
2390 *
2391 * Returns 0, this is not allowed to fail
2392 *
2393 * The close entry point is called when an interface is de-activated
2394 * by the OS. The hardware is still under the driver's control, but
2395 * needs to be disabled. A global MAC reset is issued to stop the
2396 * hardware, and all transmit and receive resources are freed.
2397 **/
2398 static int igb_close(struct net_device *netdev)
2399 {
2400 struct igb_adapter *adapter = netdev_priv(netdev);
2401
2402 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2403 igb_down(adapter);
2404
2405 igb_free_irq(adapter);
2406
2407 igb_free_all_tx_resources(adapter);
2408 igb_free_all_rx_resources(adapter);
2409
2410 return 0;
2411 }
2412
2413 /**
2414 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2415 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2416 *
2417 * Return 0 on success, negative on failure
2418 **/
2419 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2420 {
2421 struct device *dev = tx_ring->dev;
2422 int size;
2423
2424 size = sizeof(struct igb_buffer) * tx_ring->count;
2425 tx_ring->buffer_info = vmalloc(size);
2426 if (!tx_ring->buffer_info)
2427 goto err;
2428 memset(tx_ring->buffer_info, 0, size);
2429
2430 /* round up to nearest 4K */
2431 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2432 tx_ring->size = ALIGN(tx_ring->size, 4096);
2433
2434 tx_ring->desc = dma_alloc_coherent(dev,
2435 tx_ring->size,
2436 &tx_ring->dma,
2437 GFP_KERNEL);
2438
2439 if (!tx_ring->desc)
2440 goto err;
2441
2442 tx_ring->next_to_use = 0;
2443 tx_ring->next_to_clean = 0;
2444 return 0;
2445
2446 err:
2447 vfree(tx_ring->buffer_info);
2448 dev_err(dev,
2449 "Unable to allocate memory for the transmit descriptor ring\n");
2450 return -ENOMEM;
2451 }
2452
2453 /**
2454 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2455 * (Descriptors) for all queues
2456 * @adapter: board private structure
2457 *
2458 * Return 0 on success, negative on failure
2459 **/
2460 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2461 {
2462 struct pci_dev *pdev = adapter->pdev;
2463 int i, err = 0;
2464
2465 for (i = 0; i < adapter->num_tx_queues; i++) {
2466 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2467 if (err) {
2468 dev_err(&pdev->dev,
2469 "Allocation for Tx Queue %u failed\n", i);
2470 for (i--; i >= 0; i--)
2471 igb_free_tx_resources(adapter->tx_ring[i]);
2472 break;
2473 }
2474 }
2475
2476 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2477 int r_idx = i % adapter->num_tx_queues;
2478 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
2479 }
2480 return err;
2481 }
2482
2483 /**
2484 * igb_setup_tctl - configure the transmit control registers
2485 * @adapter: Board private structure
2486 **/
2487 void igb_setup_tctl(struct igb_adapter *adapter)
2488 {
2489 struct e1000_hw *hw = &adapter->hw;
2490 u32 tctl;
2491
2492 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2493 wr32(E1000_TXDCTL(0), 0);
2494
2495 /* Program the Transmit Control Register */
2496 tctl = rd32(E1000_TCTL);
2497 tctl &= ~E1000_TCTL_CT;
2498 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2499 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2500
2501 igb_config_collision_dist(hw);
2502
2503 /* Enable transmits */
2504 tctl |= E1000_TCTL_EN;
2505
2506 wr32(E1000_TCTL, tctl);
2507 }
2508
2509 /**
2510 * igb_configure_tx_ring - Configure transmit ring after Reset
2511 * @adapter: board private structure
2512 * @ring: tx ring to configure
2513 *
2514 * Configure a transmit ring after a reset.
2515 **/
2516 void igb_configure_tx_ring(struct igb_adapter *adapter,
2517 struct igb_ring *ring)
2518 {
2519 struct e1000_hw *hw = &adapter->hw;
2520 u32 txdctl;
2521 u64 tdba = ring->dma;
2522 int reg_idx = ring->reg_idx;
2523
2524 /* disable the queue */
2525 txdctl = rd32(E1000_TXDCTL(reg_idx));
2526 wr32(E1000_TXDCTL(reg_idx),
2527 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2528 wrfl();
2529 mdelay(10);
2530
2531 wr32(E1000_TDLEN(reg_idx),
2532 ring->count * sizeof(union e1000_adv_tx_desc));
2533 wr32(E1000_TDBAL(reg_idx),
2534 tdba & 0x00000000ffffffffULL);
2535 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2536
2537 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2538 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2539 writel(0, ring->head);
2540 writel(0, ring->tail);
2541
2542 txdctl |= IGB_TX_PTHRESH;
2543 txdctl |= IGB_TX_HTHRESH << 8;
2544 txdctl |= IGB_TX_WTHRESH << 16;
2545
2546 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2547 wr32(E1000_TXDCTL(reg_idx), txdctl);
2548 }
2549
2550 /**
2551 * igb_configure_tx - Configure transmit Unit after Reset
2552 * @adapter: board private structure
2553 *
2554 * Configure the Tx unit of the MAC after a reset.
2555 **/
2556 static void igb_configure_tx(struct igb_adapter *adapter)
2557 {
2558 int i;
2559
2560 for (i = 0; i < adapter->num_tx_queues; i++)
2561 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2562 }
2563
2564 /**
2565 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2566 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2567 *
2568 * Returns 0 on success, negative on failure
2569 **/
2570 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2571 {
2572 struct device *dev = rx_ring->dev;
2573 int size, desc_len;
2574
2575 size = sizeof(struct igb_buffer) * rx_ring->count;
2576 rx_ring->buffer_info = vmalloc(size);
2577 if (!rx_ring->buffer_info)
2578 goto err;
2579 memset(rx_ring->buffer_info, 0, size);
2580
2581 desc_len = sizeof(union e1000_adv_rx_desc);
2582
2583 /* Round up to nearest 4K */
2584 rx_ring->size = rx_ring->count * desc_len;
2585 rx_ring->size = ALIGN(rx_ring->size, 4096);
2586
2587 rx_ring->desc = dma_alloc_coherent(dev,
2588 rx_ring->size,
2589 &rx_ring->dma,
2590 GFP_KERNEL);
2591
2592 if (!rx_ring->desc)
2593 goto err;
2594
2595 rx_ring->next_to_clean = 0;
2596 rx_ring->next_to_use = 0;
2597
2598 return 0;
2599
2600 err:
2601 vfree(rx_ring->buffer_info);
2602 rx_ring->buffer_info = NULL;
2603 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2604 " ring\n");
2605 return -ENOMEM;
2606 }
2607
2608 /**
2609 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2610 * (Descriptors) for all queues
2611 * @adapter: board private structure
2612 *
2613 * Return 0 on success, negative on failure
2614 **/
2615 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2616 {
2617 struct pci_dev *pdev = adapter->pdev;
2618 int i, err = 0;
2619
2620 for (i = 0; i < adapter->num_rx_queues; i++) {
2621 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2622 if (err) {
2623 dev_err(&pdev->dev,
2624 "Allocation for Rx Queue %u failed\n", i);
2625 for (i--; i >= 0; i--)
2626 igb_free_rx_resources(adapter->rx_ring[i]);
2627 break;
2628 }
2629 }
2630
2631 return err;
2632 }
2633
2634 /**
2635 * igb_setup_mrqc - configure the multiple receive queue control registers
2636 * @adapter: Board private structure
2637 **/
2638 static void igb_setup_mrqc(struct igb_adapter *adapter)
2639 {
2640 struct e1000_hw *hw = &adapter->hw;
2641 u32 mrqc, rxcsum;
2642 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2643 union e1000_reta {
2644 u32 dword;
2645 u8 bytes[4];
2646 } reta;
2647 static const u8 rsshash[40] = {
2648 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2649 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2650 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2651 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2652
2653 /* Fill out hash function seeds */
2654 for (j = 0; j < 10; j++) {
2655 u32 rsskey = rsshash[(j * 4)];
2656 rsskey |= rsshash[(j * 4) + 1] << 8;
2657 rsskey |= rsshash[(j * 4) + 2] << 16;
2658 rsskey |= rsshash[(j * 4) + 3] << 24;
2659 array_wr32(E1000_RSSRK(0), j, rsskey);
2660 }
2661
2662 num_rx_queues = adapter->rss_queues;
2663
2664 if (adapter->vfs_allocated_count) {
2665 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2666 switch (hw->mac.type) {
2667 case e1000_i350:
2668 case e1000_82580:
2669 num_rx_queues = 1;
2670 shift = 0;
2671 break;
2672 case e1000_82576:
2673 shift = 3;
2674 num_rx_queues = 2;
2675 break;
2676 case e1000_82575:
2677 shift = 2;
2678 shift2 = 6;
2679 default:
2680 break;
2681 }
2682 } else {
2683 if (hw->mac.type == e1000_82575)
2684 shift = 6;
2685 }
2686
2687 for (j = 0; j < (32 * 4); j++) {
2688 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2689 if (shift2)
2690 reta.bytes[j & 3] |= num_rx_queues << shift2;
2691 if ((j & 3) == 3)
2692 wr32(E1000_RETA(j >> 2), reta.dword);
2693 }
2694
2695 /*
2696 * Disable raw packet checksumming so that RSS hash is placed in
2697 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2698 * offloads as they are enabled by default
2699 */
2700 rxcsum = rd32(E1000_RXCSUM);
2701 rxcsum |= E1000_RXCSUM_PCSD;
2702
2703 if (adapter->hw.mac.type >= e1000_82576)
2704 /* Enable Receive Checksum Offload for SCTP */
2705 rxcsum |= E1000_RXCSUM_CRCOFL;
2706
2707 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2708 wr32(E1000_RXCSUM, rxcsum);
2709
2710 /* If VMDq is enabled then we set the appropriate mode for that, else
2711 * we default to RSS so that an RSS hash is calculated per packet even
2712 * if we are only using one queue */
2713 if (adapter->vfs_allocated_count) {
2714 if (hw->mac.type > e1000_82575) {
2715 /* Set the default pool for the PF's first queue */
2716 u32 vtctl = rd32(E1000_VT_CTL);
2717 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2718 E1000_VT_CTL_DISABLE_DEF_POOL);
2719 vtctl |= adapter->vfs_allocated_count <<
2720 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2721 wr32(E1000_VT_CTL, vtctl);
2722 }
2723 if (adapter->rss_queues > 1)
2724 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2725 else
2726 mrqc = E1000_MRQC_ENABLE_VMDQ;
2727 } else {
2728 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2729 }
2730 igb_vmm_control(adapter);
2731
2732 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2733 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2734 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2735 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2736 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2737 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2738 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2739 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2740
2741 wr32(E1000_MRQC, mrqc);
2742 }
2743
2744 /**
2745 * igb_setup_rctl - configure the receive control registers
2746 * @adapter: Board private structure
2747 **/
2748 void igb_setup_rctl(struct igb_adapter *adapter)
2749 {
2750 struct e1000_hw *hw = &adapter->hw;
2751 u32 rctl;
2752
2753 rctl = rd32(E1000_RCTL);
2754
2755 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2756 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2757
2758 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2759 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2760
2761 /*
2762 * enable stripping of CRC. It's unlikely this will break BMC
2763 * redirection as it did with e1000. Newer features require
2764 * that the HW strips the CRC.
2765 */
2766 rctl |= E1000_RCTL_SECRC;
2767
2768 /* disable store bad packets and clear size bits. */
2769 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2770
2771 /* enable LPE to prevent packets larger than max_frame_size */
2772 rctl |= E1000_RCTL_LPE;
2773
2774 /* disable queue 0 to prevent tail write w/o re-config */
2775 wr32(E1000_RXDCTL(0), 0);
2776
2777 /* Attention!!! For SR-IOV PF driver operations you must enable
2778 * queue drop for all VF and PF queues to prevent head of line blocking
2779 * if an un-trusted VF does not provide descriptors to hardware.
2780 */
2781 if (adapter->vfs_allocated_count) {
2782 /* set all queue drop enable bits */
2783 wr32(E1000_QDE, ALL_QUEUES);
2784 }
2785
2786 wr32(E1000_RCTL, rctl);
2787 }
2788
2789 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2790 int vfn)
2791 {
2792 struct e1000_hw *hw = &adapter->hw;
2793 u32 vmolr;
2794
2795 /* if it isn't the PF check to see if VFs are enabled and
2796 * increase the size to support vlan tags */
2797 if (vfn < adapter->vfs_allocated_count &&
2798 adapter->vf_data[vfn].vlans_enabled)
2799 size += VLAN_TAG_SIZE;
2800
2801 vmolr = rd32(E1000_VMOLR(vfn));
2802 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2803 vmolr |= size | E1000_VMOLR_LPE;
2804 wr32(E1000_VMOLR(vfn), vmolr);
2805
2806 return 0;
2807 }
2808
2809 /**
2810 * igb_rlpml_set - set maximum receive packet size
2811 * @adapter: board private structure
2812 *
2813 * Configure maximum receivable packet size.
2814 **/
2815 static void igb_rlpml_set(struct igb_adapter *adapter)
2816 {
2817 u32 max_frame_size = adapter->max_frame_size;
2818 struct e1000_hw *hw = &adapter->hw;
2819 u16 pf_id = adapter->vfs_allocated_count;
2820
2821 if (adapter->vlgrp)
2822 max_frame_size += VLAN_TAG_SIZE;
2823
2824 /* if vfs are enabled we set RLPML to the largest possible request
2825 * size and set the VMOLR RLPML to the size we need */
2826 if (pf_id) {
2827 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2828 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2829 }
2830
2831 wr32(E1000_RLPML, max_frame_size);
2832 }
2833
2834 static inline void igb_set_vmolr(struct igb_adapter *adapter,
2835 int vfn, bool aupe)
2836 {
2837 struct e1000_hw *hw = &adapter->hw;
2838 u32 vmolr;
2839
2840 /*
2841 * This register exists only on 82576 and newer so if we are older then
2842 * we should exit and do nothing
2843 */
2844 if (hw->mac.type < e1000_82576)
2845 return;
2846
2847 vmolr = rd32(E1000_VMOLR(vfn));
2848 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2849 if (aupe)
2850 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2851 else
2852 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
2853
2854 /* clear all bits that might not be set */
2855 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2856
2857 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2858 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2859 /*
2860 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2861 * multicast packets
2862 */
2863 if (vfn <= adapter->vfs_allocated_count)
2864 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2865
2866 wr32(E1000_VMOLR(vfn), vmolr);
2867 }
2868
2869 /**
2870 * igb_configure_rx_ring - Configure a receive ring after Reset
2871 * @adapter: board private structure
2872 * @ring: receive ring to be configured
2873 *
2874 * Configure the Rx unit of the MAC after a reset.
2875 **/
2876 void igb_configure_rx_ring(struct igb_adapter *adapter,
2877 struct igb_ring *ring)
2878 {
2879 struct e1000_hw *hw = &adapter->hw;
2880 u64 rdba = ring->dma;
2881 int reg_idx = ring->reg_idx;
2882 u32 srrctl, rxdctl;
2883
2884 /* disable the queue */
2885 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2886 wr32(E1000_RXDCTL(reg_idx),
2887 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2888
2889 /* Set DMA base address registers */
2890 wr32(E1000_RDBAL(reg_idx),
2891 rdba & 0x00000000ffffffffULL);
2892 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2893 wr32(E1000_RDLEN(reg_idx),
2894 ring->count * sizeof(union e1000_adv_rx_desc));
2895
2896 /* initialize head and tail */
2897 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2898 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2899 writel(0, ring->head);
2900 writel(0, ring->tail);
2901
2902 /* set descriptor configuration */
2903 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2904 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2905 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2906 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2907 srrctl |= IGB_RXBUFFER_16384 >>
2908 E1000_SRRCTL_BSIZEPKT_SHIFT;
2909 #else
2910 srrctl |= (PAGE_SIZE / 2) >>
2911 E1000_SRRCTL_BSIZEPKT_SHIFT;
2912 #endif
2913 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2914 } else {
2915 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2916 E1000_SRRCTL_BSIZEPKT_SHIFT;
2917 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2918 }
2919 if (hw->mac.type == e1000_82580)
2920 srrctl |= E1000_SRRCTL_TIMESTAMP;
2921 /* Only set Drop Enable if we are supporting multiple queues */
2922 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2923 srrctl |= E1000_SRRCTL_DROP_EN;
2924
2925 wr32(E1000_SRRCTL(reg_idx), srrctl);
2926
2927 /* set filtering for VMDQ pools */
2928 igb_set_vmolr(adapter, reg_idx & 0x7, true);
2929
2930 /* enable receive descriptor fetching */
2931 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2932 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2933 rxdctl &= 0xFFF00000;
2934 rxdctl |= IGB_RX_PTHRESH;
2935 rxdctl |= IGB_RX_HTHRESH << 8;
2936 rxdctl |= IGB_RX_WTHRESH << 16;
2937 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2938 }
2939
2940 /**
2941 * igb_configure_rx - Configure receive Unit after Reset
2942 * @adapter: board private structure
2943 *
2944 * Configure the Rx unit of the MAC after a reset.
2945 **/
2946 static void igb_configure_rx(struct igb_adapter *adapter)
2947 {
2948 int i;
2949
2950 /* set UTA to appropriate mode */
2951 igb_set_uta(adapter);
2952
2953 /* set the correct pool for the PF default MAC address in entry 0 */
2954 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2955 adapter->vfs_allocated_count);
2956
2957 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2958 * the Base and Length of the Rx Descriptor Ring */
2959 for (i = 0; i < adapter->num_rx_queues; i++)
2960 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
2961 }
2962
2963 /**
2964 * igb_free_tx_resources - Free Tx Resources per Queue
2965 * @tx_ring: Tx descriptor ring for a specific queue
2966 *
2967 * Free all transmit software resources
2968 **/
2969 void igb_free_tx_resources(struct igb_ring *tx_ring)
2970 {
2971 igb_clean_tx_ring(tx_ring);
2972
2973 vfree(tx_ring->buffer_info);
2974 tx_ring->buffer_info = NULL;
2975
2976 /* if not set, then don't free */
2977 if (!tx_ring->desc)
2978 return;
2979
2980 dma_free_coherent(tx_ring->dev, tx_ring->size,
2981 tx_ring->desc, tx_ring->dma);
2982
2983 tx_ring->desc = NULL;
2984 }
2985
2986 /**
2987 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2988 * @adapter: board private structure
2989 *
2990 * Free all transmit software resources
2991 **/
2992 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2993 {
2994 int i;
2995
2996 for (i = 0; i < adapter->num_tx_queues; i++)
2997 igb_free_tx_resources(adapter->tx_ring[i]);
2998 }
2999
3000 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3001 struct igb_buffer *buffer_info)
3002 {
3003 if (buffer_info->dma) {
3004 if (buffer_info->mapped_as_page)
3005 dma_unmap_page(tx_ring->dev,
3006 buffer_info->dma,
3007 buffer_info->length,
3008 DMA_TO_DEVICE);
3009 else
3010 dma_unmap_single(tx_ring->dev,
3011 buffer_info->dma,
3012 buffer_info->length,
3013 DMA_TO_DEVICE);
3014 buffer_info->dma = 0;
3015 }
3016 if (buffer_info->skb) {
3017 dev_kfree_skb_any(buffer_info->skb);
3018 buffer_info->skb = NULL;
3019 }
3020 buffer_info->time_stamp = 0;
3021 buffer_info->length = 0;
3022 buffer_info->next_to_watch = 0;
3023 buffer_info->mapped_as_page = false;
3024 }
3025
3026 /**
3027 * igb_clean_tx_ring - Free Tx Buffers
3028 * @tx_ring: ring to be cleaned
3029 **/
3030 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3031 {
3032 struct igb_buffer *buffer_info;
3033 unsigned long size;
3034 unsigned int i;
3035
3036 if (!tx_ring->buffer_info)
3037 return;
3038 /* Free all the Tx ring sk_buffs */
3039
3040 for (i = 0; i < tx_ring->count; i++) {
3041 buffer_info = &tx_ring->buffer_info[i];
3042 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3043 }
3044
3045 size = sizeof(struct igb_buffer) * tx_ring->count;
3046 memset(tx_ring->buffer_info, 0, size);
3047
3048 /* Zero out the descriptor ring */
3049 memset(tx_ring->desc, 0, tx_ring->size);
3050
3051 tx_ring->next_to_use = 0;
3052 tx_ring->next_to_clean = 0;
3053 }
3054
3055 /**
3056 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3057 * @adapter: board private structure
3058 **/
3059 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3060 {
3061 int i;
3062
3063 for (i = 0; i < adapter->num_tx_queues; i++)
3064 igb_clean_tx_ring(adapter->tx_ring[i]);
3065 }
3066
3067 /**
3068 * igb_free_rx_resources - Free Rx Resources
3069 * @rx_ring: ring to clean the resources from
3070 *
3071 * Free all receive software resources
3072 **/
3073 void igb_free_rx_resources(struct igb_ring *rx_ring)
3074 {
3075 igb_clean_rx_ring(rx_ring);
3076
3077 vfree(rx_ring->buffer_info);
3078 rx_ring->buffer_info = NULL;
3079
3080 /* if not set, then don't free */
3081 if (!rx_ring->desc)
3082 return;
3083
3084 dma_free_coherent(rx_ring->dev, rx_ring->size,
3085 rx_ring->desc, rx_ring->dma);
3086
3087 rx_ring->desc = NULL;
3088 }
3089
3090 /**
3091 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3092 * @adapter: board private structure
3093 *
3094 * Free all receive software resources
3095 **/
3096 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3097 {
3098 int i;
3099
3100 for (i = 0; i < adapter->num_rx_queues; i++)
3101 igb_free_rx_resources(adapter->rx_ring[i]);
3102 }
3103
3104 /**
3105 * igb_clean_rx_ring - Free Rx Buffers per Queue
3106 * @rx_ring: ring to free buffers from
3107 **/
3108 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3109 {
3110 struct igb_buffer *buffer_info;
3111 unsigned long size;
3112 unsigned int i;
3113
3114 if (!rx_ring->buffer_info)
3115 return;
3116
3117 /* Free all the Rx ring sk_buffs */
3118 for (i = 0; i < rx_ring->count; i++) {
3119 buffer_info = &rx_ring->buffer_info[i];
3120 if (buffer_info->dma) {
3121 dma_unmap_single(rx_ring->dev,
3122 buffer_info->dma,
3123 rx_ring->rx_buffer_len,
3124 DMA_FROM_DEVICE);
3125 buffer_info->dma = 0;
3126 }
3127
3128 if (buffer_info->skb) {
3129 dev_kfree_skb(buffer_info->skb);
3130 buffer_info->skb = NULL;
3131 }
3132 if (buffer_info->page_dma) {
3133 dma_unmap_page(rx_ring->dev,
3134 buffer_info->page_dma,
3135 PAGE_SIZE / 2,
3136 DMA_FROM_DEVICE);
3137 buffer_info->page_dma = 0;
3138 }
3139 if (buffer_info->page) {
3140 put_page(buffer_info->page);
3141 buffer_info->page = NULL;
3142 buffer_info->page_offset = 0;
3143 }
3144 }
3145
3146 size = sizeof(struct igb_buffer) * rx_ring->count;
3147 memset(rx_ring->buffer_info, 0, size);
3148
3149 /* Zero out the descriptor ring */
3150 memset(rx_ring->desc, 0, rx_ring->size);
3151
3152 rx_ring->next_to_clean = 0;
3153 rx_ring->next_to_use = 0;
3154 }
3155
3156 /**
3157 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3158 * @adapter: board private structure
3159 **/
3160 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3161 {
3162 int i;
3163
3164 for (i = 0; i < adapter->num_rx_queues; i++)
3165 igb_clean_rx_ring(adapter->rx_ring[i]);
3166 }
3167
3168 /**
3169 * igb_set_mac - Change the Ethernet Address of the NIC
3170 * @netdev: network interface device structure
3171 * @p: pointer to an address structure
3172 *
3173 * Returns 0 on success, negative on failure
3174 **/
3175 static int igb_set_mac(struct net_device *netdev, void *p)
3176 {
3177 struct igb_adapter *adapter = netdev_priv(netdev);
3178 struct e1000_hw *hw = &adapter->hw;
3179 struct sockaddr *addr = p;
3180
3181 if (!is_valid_ether_addr(addr->sa_data))
3182 return -EADDRNOTAVAIL;
3183
3184 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3185 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3186
3187 /* set the correct pool for the new PF MAC address in entry 0 */
3188 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3189 adapter->vfs_allocated_count);
3190
3191 return 0;
3192 }
3193
3194 /**
3195 * igb_write_mc_addr_list - write multicast addresses to MTA
3196 * @netdev: network interface device structure
3197 *
3198 * Writes multicast address list to the MTA hash table.
3199 * Returns: -ENOMEM on failure
3200 * 0 on no addresses written
3201 * X on writing X addresses to MTA
3202 **/
3203 static int igb_write_mc_addr_list(struct net_device *netdev)
3204 {
3205 struct igb_adapter *adapter = netdev_priv(netdev);
3206 struct e1000_hw *hw = &adapter->hw;
3207 struct netdev_hw_addr *ha;
3208 u8 *mta_list;
3209 int i;
3210
3211 if (netdev_mc_empty(netdev)) {
3212 /* nothing to program, so clear mc list */
3213 igb_update_mc_addr_list(hw, NULL, 0);
3214 igb_restore_vf_multicasts(adapter);
3215 return 0;
3216 }
3217
3218 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3219 if (!mta_list)
3220 return -ENOMEM;
3221
3222 /* The shared function expects a packed array of only addresses. */
3223 i = 0;
3224 netdev_for_each_mc_addr(ha, netdev)
3225 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3226
3227 igb_update_mc_addr_list(hw, mta_list, i);
3228 kfree(mta_list);
3229
3230 return netdev_mc_count(netdev);
3231 }
3232
3233 /**
3234 * igb_write_uc_addr_list - write unicast addresses to RAR table
3235 * @netdev: network interface device structure
3236 *
3237 * Writes unicast address list to the RAR table.
3238 * Returns: -ENOMEM on failure/insufficient address space
3239 * 0 on no addresses written
3240 * X on writing X addresses to the RAR table
3241 **/
3242 static int igb_write_uc_addr_list(struct net_device *netdev)
3243 {
3244 struct igb_adapter *adapter = netdev_priv(netdev);
3245 struct e1000_hw *hw = &adapter->hw;
3246 unsigned int vfn = adapter->vfs_allocated_count;
3247 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3248 int count = 0;
3249
3250 /* return ENOMEM indicating insufficient memory for addresses */
3251 if (netdev_uc_count(netdev) > rar_entries)
3252 return -ENOMEM;
3253
3254 if (!netdev_uc_empty(netdev) && rar_entries) {
3255 struct netdev_hw_addr *ha;
3256
3257 netdev_for_each_uc_addr(ha, netdev) {
3258 if (!rar_entries)
3259 break;
3260 igb_rar_set_qsel(adapter, ha->addr,
3261 rar_entries--,
3262 vfn);
3263 count++;
3264 }
3265 }
3266 /* write the addresses in reverse order to avoid write combining */
3267 for (; rar_entries > 0 ; rar_entries--) {
3268 wr32(E1000_RAH(rar_entries), 0);
3269 wr32(E1000_RAL(rar_entries), 0);
3270 }
3271 wrfl();
3272
3273 return count;
3274 }
3275
3276 /**
3277 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3278 * @netdev: network interface device structure
3279 *
3280 * The set_rx_mode entry point is called whenever the unicast or multicast
3281 * address lists or the network interface flags are updated. This routine is
3282 * responsible for configuring the hardware for proper unicast, multicast,
3283 * promiscuous mode, and all-multi behavior.
3284 **/
3285 static void igb_set_rx_mode(struct net_device *netdev)
3286 {
3287 struct igb_adapter *adapter = netdev_priv(netdev);
3288 struct e1000_hw *hw = &adapter->hw;
3289 unsigned int vfn = adapter->vfs_allocated_count;
3290 u32 rctl, vmolr = 0;
3291 int count;
3292
3293 /* Check for Promiscuous and All Multicast modes */
3294 rctl = rd32(E1000_RCTL);
3295
3296 /* clear the effected bits */
3297 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3298
3299 if (netdev->flags & IFF_PROMISC) {
3300 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3301 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3302 } else {
3303 if (netdev->flags & IFF_ALLMULTI) {
3304 rctl |= E1000_RCTL_MPE;
3305 vmolr |= E1000_VMOLR_MPME;
3306 } else {
3307 /*
3308 * Write addresses to the MTA, if the attempt fails
3309 * then we should just turn on promiscous mode so
3310 * that we can at least receive multicast traffic
3311 */
3312 count = igb_write_mc_addr_list(netdev);
3313 if (count < 0) {
3314 rctl |= E1000_RCTL_MPE;
3315 vmolr |= E1000_VMOLR_MPME;
3316 } else if (count) {
3317 vmolr |= E1000_VMOLR_ROMPE;
3318 }
3319 }
3320 /*
3321 * Write addresses to available RAR registers, if there is not
3322 * sufficient space to store all the addresses then enable
3323 * unicast promiscous mode
3324 */
3325 count = igb_write_uc_addr_list(netdev);
3326 if (count < 0) {
3327 rctl |= E1000_RCTL_UPE;
3328 vmolr |= E1000_VMOLR_ROPE;
3329 }
3330 rctl |= E1000_RCTL_VFE;
3331 }
3332 wr32(E1000_RCTL, rctl);
3333
3334 /*
3335 * In order to support SR-IOV and eventually VMDq it is necessary to set
3336 * the VMOLR to enable the appropriate modes. Without this workaround
3337 * we will have issues with VLAN tag stripping not being done for frames
3338 * that are only arriving because we are the default pool
3339 */
3340 if (hw->mac.type < e1000_82576)
3341 return;
3342
3343 vmolr |= rd32(E1000_VMOLR(vfn)) &
3344 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3345 wr32(E1000_VMOLR(vfn), vmolr);
3346 igb_restore_vf_multicasts(adapter);
3347 }
3348
3349 /* Need to wait a few seconds after link up to get diagnostic information from
3350 * the phy */
3351 static void igb_update_phy_info(unsigned long data)
3352 {
3353 struct igb_adapter *adapter = (struct igb_adapter *) data;
3354 igb_get_phy_info(&adapter->hw);
3355 }
3356
3357 /**
3358 * igb_has_link - check shared code for link and determine up/down
3359 * @adapter: pointer to driver private info
3360 **/
3361 bool igb_has_link(struct igb_adapter *adapter)
3362 {
3363 struct e1000_hw *hw = &adapter->hw;
3364 bool link_active = false;
3365 s32 ret_val = 0;
3366
3367 /* get_link_status is set on LSC (link status) interrupt or
3368 * rx sequence error interrupt. get_link_status will stay
3369 * false until the e1000_check_for_link establishes link
3370 * for copper adapters ONLY
3371 */
3372 switch (hw->phy.media_type) {
3373 case e1000_media_type_copper:
3374 if (hw->mac.get_link_status) {
3375 ret_val = hw->mac.ops.check_for_link(hw);
3376 link_active = !hw->mac.get_link_status;
3377 } else {
3378 link_active = true;
3379 }
3380 break;
3381 case e1000_media_type_internal_serdes:
3382 ret_val = hw->mac.ops.check_for_link(hw);
3383 link_active = hw->mac.serdes_has_link;
3384 break;
3385 default:
3386 case e1000_media_type_unknown:
3387 break;
3388 }
3389
3390 return link_active;
3391 }
3392
3393 /**
3394 * igb_watchdog - Timer Call-back
3395 * @data: pointer to adapter cast into an unsigned long
3396 **/
3397 static void igb_watchdog(unsigned long data)
3398 {
3399 struct igb_adapter *adapter = (struct igb_adapter *)data;
3400 /* Do the rest outside of interrupt context */
3401 schedule_work(&adapter->watchdog_task);
3402 }
3403
3404 static void igb_watchdog_task(struct work_struct *work)
3405 {
3406 struct igb_adapter *adapter = container_of(work,
3407 struct igb_adapter,
3408 watchdog_task);
3409 struct e1000_hw *hw = &adapter->hw;
3410 struct net_device *netdev = adapter->netdev;
3411 u32 link;
3412 int i;
3413
3414 link = igb_has_link(adapter);
3415 if (link) {
3416 if (!netif_carrier_ok(netdev)) {
3417 u32 ctrl;
3418 hw->mac.ops.get_speed_and_duplex(hw,
3419 &adapter->link_speed,
3420 &adapter->link_duplex);
3421
3422 ctrl = rd32(E1000_CTRL);
3423 /* Links status message must follow this format */
3424 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3425 "Flow Control: %s\n",
3426 netdev->name,
3427 adapter->link_speed,
3428 adapter->link_duplex == FULL_DUPLEX ?
3429 "Full Duplex" : "Half Duplex",
3430 ((ctrl & E1000_CTRL_TFCE) &&
3431 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3432 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3433 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
3434
3435 /* adjust timeout factor according to speed/duplex */
3436 adapter->tx_timeout_factor = 1;
3437 switch (adapter->link_speed) {
3438 case SPEED_10:
3439 adapter->tx_timeout_factor = 14;
3440 break;
3441 case SPEED_100:
3442 /* maybe add some timeout factor ? */
3443 break;
3444 }
3445
3446 netif_carrier_on(netdev);
3447
3448 igb_ping_all_vfs(adapter);
3449
3450 /* link state has changed, schedule phy info update */
3451 if (!test_bit(__IGB_DOWN, &adapter->state))
3452 mod_timer(&adapter->phy_info_timer,
3453 round_jiffies(jiffies + 2 * HZ));
3454 }
3455 } else {
3456 if (netif_carrier_ok(netdev)) {
3457 adapter->link_speed = 0;
3458 adapter->link_duplex = 0;
3459 /* Links status message must follow this format */
3460 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3461 netdev->name);
3462 netif_carrier_off(netdev);
3463
3464 igb_ping_all_vfs(adapter);
3465
3466 /* link state has changed, schedule phy info update */
3467 if (!test_bit(__IGB_DOWN, &adapter->state))
3468 mod_timer(&adapter->phy_info_timer,
3469 round_jiffies(jiffies + 2 * HZ));
3470 }
3471 }
3472
3473 igb_update_stats(adapter);
3474
3475 for (i = 0; i < adapter->num_tx_queues; i++) {
3476 struct igb_ring *tx_ring = adapter->tx_ring[i];
3477 if (!netif_carrier_ok(netdev)) {
3478 /* We've lost link, so the controller stops DMA,
3479 * but we've got queued Tx work that's never going
3480 * to get done, so reset controller to flush Tx.
3481 * (Do the reset outside of interrupt context). */
3482 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3483 adapter->tx_timeout_count++;
3484 schedule_work(&adapter->reset_task);
3485 /* return immediately since reset is imminent */
3486 return;
3487 }
3488 }
3489
3490 /* Force detection of hung controller every watchdog period */
3491 tx_ring->detect_tx_hung = true;
3492 }
3493
3494 /* Cause software interrupt to ensure rx ring is cleaned */
3495 if (adapter->msix_entries) {
3496 u32 eics = 0;
3497 for (i = 0; i < adapter->num_q_vectors; i++) {
3498 struct igb_q_vector *q_vector = adapter->q_vector[i];
3499 eics |= q_vector->eims_value;
3500 }
3501 wr32(E1000_EICS, eics);
3502 } else {
3503 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3504 }
3505
3506 /* Reset the timer */
3507 if (!test_bit(__IGB_DOWN, &adapter->state))
3508 mod_timer(&adapter->watchdog_timer,
3509 round_jiffies(jiffies + 2 * HZ));
3510 }
3511
3512 enum latency_range {
3513 lowest_latency = 0,
3514 low_latency = 1,
3515 bulk_latency = 2,
3516 latency_invalid = 255
3517 };
3518
3519 /**
3520 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3521 *
3522 * Stores a new ITR value based on strictly on packet size. This
3523 * algorithm is less sophisticated than that used in igb_update_itr,
3524 * due to the difficulty of synchronizing statistics across multiple
3525 * receive rings. The divisors and thresholds used by this fuction
3526 * were determined based on theoretical maximum wire speed and testing
3527 * data, in order to minimize response time while increasing bulk
3528 * throughput.
3529 * This functionality is controlled by the InterruptThrottleRate module
3530 * parameter (see igb_param.c)
3531 * NOTE: This function is called only when operating in a multiqueue
3532 * receive environment.
3533 * @q_vector: pointer to q_vector
3534 **/
3535 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3536 {
3537 int new_val = q_vector->itr_val;
3538 int avg_wire_size = 0;
3539 struct igb_adapter *adapter = q_vector->adapter;
3540
3541 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3542 * ints/sec - ITR timer value of 120 ticks.
3543 */
3544 if (adapter->link_speed != SPEED_1000) {
3545 new_val = 976;
3546 goto set_itr_val;
3547 }
3548
3549 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3550 struct igb_ring *ring = q_vector->rx_ring;
3551 avg_wire_size = ring->total_bytes / ring->total_packets;
3552 }
3553
3554 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3555 struct igb_ring *ring = q_vector->tx_ring;
3556 avg_wire_size = max_t(u32, avg_wire_size,
3557 (ring->total_bytes /
3558 ring->total_packets));
3559 }
3560
3561 /* if avg_wire_size isn't set no work was done */
3562 if (!avg_wire_size)
3563 goto clear_counts;
3564
3565 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3566 avg_wire_size += 24;
3567
3568 /* Don't starve jumbo frames */
3569 avg_wire_size = min(avg_wire_size, 3000);
3570
3571 /* Give a little boost to mid-size frames */
3572 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3573 new_val = avg_wire_size / 3;
3574 else
3575 new_val = avg_wire_size / 2;
3576
3577 /* when in itr mode 3 do not exceed 20K ints/sec */
3578 if (adapter->rx_itr_setting == 3 && new_val < 196)
3579 new_val = 196;
3580
3581 set_itr_val:
3582 if (new_val != q_vector->itr_val) {
3583 q_vector->itr_val = new_val;
3584 q_vector->set_itr = 1;
3585 }
3586 clear_counts:
3587 if (q_vector->rx_ring) {
3588 q_vector->rx_ring->total_bytes = 0;
3589 q_vector->rx_ring->total_packets = 0;
3590 }
3591 if (q_vector->tx_ring) {
3592 q_vector->tx_ring->total_bytes = 0;
3593 q_vector->tx_ring->total_packets = 0;
3594 }
3595 }
3596
3597 /**
3598 * igb_update_itr - update the dynamic ITR value based on statistics
3599 * Stores a new ITR value based on packets and byte
3600 * counts during the last interrupt. The advantage of per interrupt
3601 * computation is faster updates and more accurate ITR for the current
3602 * traffic pattern. Constants in this function were computed
3603 * based on theoretical maximum wire speed and thresholds were set based
3604 * on testing data as well as attempting to minimize response time
3605 * while increasing bulk throughput.
3606 * this functionality is controlled by the InterruptThrottleRate module
3607 * parameter (see igb_param.c)
3608 * NOTE: These calculations are only valid when operating in a single-
3609 * queue environment.
3610 * @adapter: pointer to adapter
3611 * @itr_setting: current q_vector->itr_val
3612 * @packets: the number of packets during this measurement interval
3613 * @bytes: the number of bytes during this measurement interval
3614 **/
3615 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3616 int packets, int bytes)
3617 {
3618 unsigned int retval = itr_setting;
3619
3620 if (packets == 0)
3621 goto update_itr_done;
3622
3623 switch (itr_setting) {
3624 case lowest_latency:
3625 /* handle TSO and jumbo frames */
3626 if (bytes/packets > 8000)
3627 retval = bulk_latency;
3628 else if ((packets < 5) && (bytes > 512))
3629 retval = low_latency;
3630 break;
3631 case low_latency: /* 50 usec aka 20000 ints/s */
3632 if (bytes > 10000) {
3633 /* this if handles the TSO accounting */
3634 if (bytes/packets > 8000) {
3635 retval = bulk_latency;
3636 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3637 retval = bulk_latency;
3638 } else if ((packets > 35)) {
3639 retval = lowest_latency;
3640 }
3641 } else if (bytes/packets > 2000) {
3642 retval = bulk_latency;
3643 } else if (packets <= 2 && bytes < 512) {
3644 retval = lowest_latency;
3645 }
3646 break;
3647 case bulk_latency: /* 250 usec aka 4000 ints/s */
3648 if (bytes > 25000) {
3649 if (packets > 35)
3650 retval = low_latency;
3651 } else if (bytes < 1500) {
3652 retval = low_latency;
3653 }
3654 break;
3655 }
3656
3657 update_itr_done:
3658 return retval;
3659 }
3660
3661 static void igb_set_itr(struct igb_adapter *adapter)
3662 {
3663 struct igb_q_vector *q_vector = adapter->q_vector[0];
3664 u16 current_itr;
3665 u32 new_itr = q_vector->itr_val;
3666
3667 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3668 if (adapter->link_speed != SPEED_1000) {
3669 current_itr = 0;
3670 new_itr = 4000;
3671 goto set_itr_now;
3672 }
3673
3674 adapter->rx_itr = igb_update_itr(adapter,
3675 adapter->rx_itr,
3676 q_vector->rx_ring->total_packets,
3677 q_vector->rx_ring->total_bytes);
3678
3679 adapter->tx_itr = igb_update_itr(adapter,
3680 adapter->tx_itr,
3681 q_vector->tx_ring->total_packets,
3682 q_vector->tx_ring->total_bytes);
3683 current_itr = max(adapter->rx_itr, adapter->tx_itr);
3684
3685 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3686 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3687 current_itr = low_latency;
3688
3689 switch (current_itr) {
3690 /* counts and packets in update_itr are dependent on these numbers */
3691 case lowest_latency:
3692 new_itr = 56; /* aka 70,000 ints/sec */
3693 break;
3694 case low_latency:
3695 new_itr = 196; /* aka 20,000 ints/sec */
3696 break;
3697 case bulk_latency:
3698 new_itr = 980; /* aka 4,000 ints/sec */
3699 break;
3700 default:
3701 break;
3702 }
3703
3704 set_itr_now:
3705 q_vector->rx_ring->total_bytes = 0;
3706 q_vector->rx_ring->total_packets = 0;
3707 q_vector->tx_ring->total_bytes = 0;
3708 q_vector->tx_ring->total_packets = 0;
3709
3710 if (new_itr != q_vector->itr_val) {
3711 /* this attempts to bias the interrupt rate towards Bulk
3712 * by adding intermediate steps when interrupt rate is
3713 * increasing */
3714 new_itr = new_itr > q_vector->itr_val ?
3715 max((new_itr * q_vector->itr_val) /
3716 (new_itr + (q_vector->itr_val >> 2)),
3717 new_itr) :
3718 new_itr;
3719 /* Don't write the value here; it resets the adapter's
3720 * internal timer, and causes us to delay far longer than
3721 * we should between interrupts. Instead, we write the ITR
3722 * value at the beginning of the next interrupt so the timing
3723 * ends up being correct.
3724 */
3725 q_vector->itr_val = new_itr;
3726 q_vector->set_itr = 1;
3727 }
3728 }
3729
3730 #define IGB_TX_FLAGS_CSUM 0x00000001
3731 #define IGB_TX_FLAGS_VLAN 0x00000002
3732 #define IGB_TX_FLAGS_TSO 0x00000004
3733 #define IGB_TX_FLAGS_IPV4 0x00000008
3734 #define IGB_TX_FLAGS_TSTAMP 0x00000010
3735 #define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3736 #define IGB_TX_FLAGS_VLAN_SHIFT 16
3737
3738 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3739 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3740 {
3741 struct e1000_adv_tx_context_desc *context_desc;
3742 unsigned int i;
3743 int err;
3744 struct igb_buffer *buffer_info;
3745 u32 info = 0, tu_cmd = 0;
3746 u32 mss_l4len_idx;
3747 u8 l4len;
3748
3749 if (skb_header_cloned(skb)) {
3750 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3751 if (err)
3752 return err;
3753 }
3754
3755 l4len = tcp_hdrlen(skb);
3756 *hdr_len += l4len;
3757
3758 if (skb->protocol == htons(ETH_P_IP)) {
3759 struct iphdr *iph = ip_hdr(skb);
3760 iph->tot_len = 0;
3761 iph->check = 0;
3762 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3763 iph->daddr, 0,
3764 IPPROTO_TCP,
3765 0);
3766 } else if (skb_is_gso_v6(skb)) {
3767 ipv6_hdr(skb)->payload_len = 0;
3768 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3769 &ipv6_hdr(skb)->daddr,
3770 0, IPPROTO_TCP, 0);
3771 }
3772
3773 i = tx_ring->next_to_use;
3774
3775 buffer_info = &tx_ring->buffer_info[i];
3776 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3777 /* VLAN MACLEN IPLEN */
3778 if (tx_flags & IGB_TX_FLAGS_VLAN)
3779 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3780 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3781 *hdr_len += skb_network_offset(skb);
3782 info |= skb_network_header_len(skb);
3783 *hdr_len += skb_network_header_len(skb);
3784 context_desc->vlan_macip_lens = cpu_to_le32(info);
3785
3786 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3787 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3788
3789 if (skb->protocol == htons(ETH_P_IP))
3790 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3791 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3792
3793 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3794
3795 /* MSS L4LEN IDX */
3796 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3797 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3798
3799 /* For 82575, context index must be unique per ring. */
3800 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3801 mss_l4len_idx |= tx_ring->reg_idx << 4;
3802
3803 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3804 context_desc->seqnum_seed = 0;
3805
3806 buffer_info->time_stamp = jiffies;
3807 buffer_info->next_to_watch = i;
3808 buffer_info->dma = 0;
3809 i++;
3810 if (i == tx_ring->count)
3811 i = 0;
3812
3813 tx_ring->next_to_use = i;
3814
3815 return true;
3816 }
3817
3818 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3819 struct sk_buff *skb, u32 tx_flags)
3820 {
3821 struct e1000_adv_tx_context_desc *context_desc;
3822 struct device *dev = tx_ring->dev;
3823 struct igb_buffer *buffer_info;
3824 u32 info = 0, tu_cmd = 0;
3825 unsigned int i;
3826
3827 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3828 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3829 i = tx_ring->next_to_use;
3830 buffer_info = &tx_ring->buffer_info[i];
3831 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3832
3833 if (tx_flags & IGB_TX_FLAGS_VLAN)
3834 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3835
3836 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3837 if (skb->ip_summed == CHECKSUM_PARTIAL)
3838 info |= skb_network_header_len(skb);
3839
3840 context_desc->vlan_macip_lens = cpu_to_le32(info);
3841
3842 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3843
3844 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3845 __be16 protocol;
3846
3847 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3848 const struct vlan_ethhdr *vhdr =
3849 (const struct vlan_ethhdr*)skb->data;
3850
3851 protocol = vhdr->h_vlan_encapsulated_proto;
3852 } else {
3853 protocol = skb->protocol;
3854 }
3855
3856 switch (protocol) {
3857 case cpu_to_be16(ETH_P_IP):
3858 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3859 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3860 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3861 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3862 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3863 break;
3864 case cpu_to_be16(ETH_P_IPV6):
3865 /* XXX what about other V6 headers?? */
3866 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3867 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3868 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3869 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3870 break;
3871 default:
3872 if (unlikely(net_ratelimit()))
3873 dev_warn(dev,
3874 "partial checksum but proto=%x!\n",
3875 skb->protocol);
3876 break;
3877 }
3878 }
3879
3880 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3881 context_desc->seqnum_seed = 0;
3882 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3883 context_desc->mss_l4len_idx =
3884 cpu_to_le32(tx_ring->reg_idx << 4);
3885
3886 buffer_info->time_stamp = jiffies;
3887 buffer_info->next_to_watch = i;
3888 buffer_info->dma = 0;
3889
3890 i++;
3891 if (i == tx_ring->count)
3892 i = 0;
3893 tx_ring->next_to_use = i;
3894
3895 return true;
3896 }
3897 return false;
3898 }
3899
3900 #define IGB_MAX_TXD_PWR 16
3901 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3902
3903 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3904 unsigned int first)
3905 {
3906 struct igb_buffer *buffer_info;
3907 struct device *dev = tx_ring->dev;
3908 unsigned int hlen = skb_headlen(skb);
3909 unsigned int count = 0, i;
3910 unsigned int f;
3911 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
3912
3913 i = tx_ring->next_to_use;
3914
3915 buffer_info = &tx_ring->buffer_info[i];
3916 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3917 buffer_info->length = hlen;
3918 /* set time_stamp *before* dma to help avoid a possible race */
3919 buffer_info->time_stamp = jiffies;
3920 buffer_info->next_to_watch = i;
3921 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
3922 DMA_TO_DEVICE);
3923 if (dma_mapping_error(dev, buffer_info->dma))
3924 goto dma_error;
3925
3926 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3927 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3928 unsigned int len = frag->size;
3929
3930 count++;
3931 i++;
3932 if (i == tx_ring->count)
3933 i = 0;
3934
3935 buffer_info = &tx_ring->buffer_info[i];
3936 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3937 buffer_info->length = len;
3938 buffer_info->time_stamp = jiffies;
3939 buffer_info->next_to_watch = i;
3940 buffer_info->mapped_as_page = true;
3941 buffer_info->dma = dma_map_page(dev,
3942 frag->page,
3943 frag->page_offset,
3944 len,
3945 DMA_TO_DEVICE);
3946 if (dma_mapping_error(dev, buffer_info->dma))
3947 goto dma_error;
3948
3949 }
3950
3951 tx_ring->buffer_info[i].skb = skb;
3952 tx_ring->buffer_info[i].shtx = skb_shinfo(skb)->tx_flags;
3953 /* multiply data chunks by size of headers */
3954 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3955 tx_ring->buffer_info[i].gso_segs = gso_segs;
3956 tx_ring->buffer_info[first].next_to_watch = i;
3957
3958 return ++count;
3959
3960 dma_error:
3961 dev_err(dev, "TX DMA map failed\n");
3962
3963 /* clear timestamp and dma mappings for failed buffer_info mapping */
3964 buffer_info->dma = 0;
3965 buffer_info->time_stamp = 0;
3966 buffer_info->length = 0;
3967 buffer_info->next_to_watch = 0;
3968 buffer_info->mapped_as_page = false;
3969
3970 /* clear timestamp and dma mappings for remaining portion of packet */
3971 while (count--) {
3972 if (i == 0)
3973 i = tx_ring->count;
3974 i--;
3975 buffer_info = &tx_ring->buffer_info[i];
3976 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3977 }
3978
3979 return 0;
3980 }
3981
3982 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
3983 u32 tx_flags, int count, u32 paylen,
3984 u8 hdr_len)
3985 {
3986 union e1000_adv_tx_desc *tx_desc;
3987 struct igb_buffer *buffer_info;
3988 u32 olinfo_status = 0, cmd_type_len;
3989 unsigned int i = tx_ring->next_to_use;
3990
3991 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3992 E1000_ADVTXD_DCMD_DEXT);
3993
3994 if (tx_flags & IGB_TX_FLAGS_VLAN)
3995 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3996
3997 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3998 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3999
4000 if (tx_flags & IGB_TX_FLAGS_TSO) {
4001 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4002
4003 /* insert tcp checksum */
4004 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4005
4006 /* insert ip checksum */
4007 if (tx_flags & IGB_TX_FLAGS_IPV4)
4008 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4009
4010 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4011 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4012 }
4013
4014 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4015 (tx_flags & (IGB_TX_FLAGS_CSUM |
4016 IGB_TX_FLAGS_TSO |
4017 IGB_TX_FLAGS_VLAN)))
4018 olinfo_status |= tx_ring->reg_idx << 4;
4019
4020 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4021
4022 do {
4023 buffer_info = &tx_ring->buffer_info[i];
4024 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4025 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4026 tx_desc->read.cmd_type_len =
4027 cpu_to_le32(cmd_type_len | buffer_info->length);
4028 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4029 count--;
4030 i++;
4031 if (i == tx_ring->count)
4032 i = 0;
4033 } while (count > 0);
4034
4035 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
4036 /* Force memory writes to complete before letting h/w
4037 * know there are new descriptors to fetch. (Only
4038 * applicable for weak-ordered memory model archs,
4039 * such as IA-64). */
4040 wmb();
4041
4042 tx_ring->next_to_use = i;
4043 writel(i, tx_ring->tail);
4044 /* we need this if more than one processor can write to our tail
4045 * at a time, it syncronizes IO on IA64/Altix systems */
4046 mmiowb();
4047 }
4048
4049 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4050 {
4051 struct net_device *netdev = tx_ring->netdev;
4052
4053 netif_stop_subqueue(netdev, tx_ring->queue_index);
4054
4055 /* Herbert's original patch had:
4056 * smp_mb__after_netif_stop_queue();
4057 * but since that doesn't exist yet, just open code it. */
4058 smp_mb();
4059
4060 /* We need to check again in a case another CPU has just
4061 * made room available. */
4062 if (igb_desc_unused(tx_ring) < size)
4063 return -EBUSY;
4064
4065 /* A reprieve! */
4066 netif_wake_subqueue(netdev, tx_ring->queue_index);
4067 tx_ring->tx_stats.restart_queue++;
4068 return 0;
4069 }
4070
4071 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4072 {
4073 if (igb_desc_unused(tx_ring) >= size)
4074 return 0;
4075 return __igb_maybe_stop_tx(tx_ring, size);
4076 }
4077
4078 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4079 struct igb_ring *tx_ring)
4080 {
4081 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
4082 int tso = 0, count;
4083 u32 tx_flags = 0;
4084 u16 first;
4085 u8 hdr_len = 0;
4086 union skb_shared_tx *shtx = skb_tx(skb);
4087
4088 /* need: 1 descriptor per page,
4089 * + 2 desc gap to keep tail from touching head,
4090 * + 1 desc for skb->data,
4091 * + 1 desc for context descriptor,
4092 * otherwise try next time */
4093 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4094 /* this is a hard error */
4095 return NETDEV_TX_BUSY;
4096 }
4097
4098 if (unlikely(shtx->hardware)) {
4099 shtx->in_progress = 1;
4100 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4101 }
4102
4103 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
4104 tx_flags |= IGB_TX_FLAGS_VLAN;
4105 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4106 }
4107
4108 if (skb->protocol == htons(ETH_P_IP))
4109 tx_flags |= IGB_TX_FLAGS_IPV4;
4110
4111 first = tx_ring->next_to_use;
4112 if (skb_is_gso(skb)) {
4113 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
4114
4115 if (tso < 0) {
4116 dev_kfree_skb_any(skb);
4117 return NETDEV_TX_OK;
4118 }
4119 }
4120
4121 if (tso)
4122 tx_flags |= IGB_TX_FLAGS_TSO;
4123 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
4124 (skb->ip_summed == CHECKSUM_PARTIAL))
4125 tx_flags |= IGB_TX_FLAGS_CSUM;
4126
4127 /*
4128 * count reflects descriptors mapped, if 0 or less then mapping error
4129 * has occured and we need to rewind the descriptor queue
4130 */
4131 count = igb_tx_map_adv(tx_ring, skb, first);
4132 if (!count) {
4133 dev_kfree_skb_any(skb);
4134 tx_ring->buffer_info[first].time_stamp = 0;
4135 tx_ring->next_to_use = first;
4136 return NETDEV_TX_OK;
4137 }
4138
4139 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4140
4141 /* Make sure there is space in the ring for the next send. */
4142 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4143
4144 return NETDEV_TX_OK;
4145 }
4146
4147 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4148 struct net_device *netdev)
4149 {
4150 struct igb_adapter *adapter = netdev_priv(netdev);
4151 struct igb_ring *tx_ring;
4152 int r_idx = 0;
4153
4154 if (test_bit(__IGB_DOWN, &adapter->state)) {
4155 dev_kfree_skb_any(skb);
4156 return NETDEV_TX_OK;
4157 }
4158
4159 if (skb->len <= 0) {
4160 dev_kfree_skb_any(skb);
4161 return NETDEV_TX_OK;
4162 }
4163
4164 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
4165 tx_ring = adapter->multi_tx_table[r_idx];
4166
4167 /* This goes back to the question of how to logically map a tx queue
4168 * to a flow. Right now, performance is impacted slightly negatively
4169 * if using multiple tx queues. If the stack breaks away from a
4170 * single qdisc implementation, we can look at this again. */
4171 return igb_xmit_frame_ring_adv(skb, tx_ring);
4172 }
4173
4174 /**
4175 * igb_tx_timeout - Respond to a Tx Hang
4176 * @netdev: network interface device structure
4177 **/
4178 static void igb_tx_timeout(struct net_device *netdev)
4179 {
4180 struct igb_adapter *adapter = netdev_priv(netdev);
4181 struct e1000_hw *hw = &adapter->hw;
4182
4183 /* Do the reset outside of interrupt context */
4184 adapter->tx_timeout_count++;
4185
4186 if (hw->mac.type == e1000_82580)
4187 hw->dev_spec._82575.global_device_reset = true;
4188
4189 schedule_work(&adapter->reset_task);
4190 wr32(E1000_EICS,
4191 (adapter->eims_enable_mask & ~adapter->eims_other));
4192 }
4193
4194 static void igb_reset_task(struct work_struct *work)
4195 {
4196 struct igb_adapter *adapter;
4197 adapter = container_of(work, struct igb_adapter, reset_task);
4198
4199 igb_dump(adapter);
4200 netdev_err(adapter->netdev, "Reset adapter\n");
4201 igb_reinit_locked(adapter);
4202 }
4203
4204 /**
4205 * igb_get_stats - Get System Network Statistics
4206 * @netdev: network interface device structure
4207 *
4208 * Returns the address of the device statistics structure.
4209 * The statistics are actually updated from the timer callback.
4210 **/
4211 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
4212 {
4213 /* only return the current stats */
4214 return &netdev->stats;
4215 }
4216
4217 /**
4218 * igb_change_mtu - Change the Maximum Transfer Unit
4219 * @netdev: network interface device structure
4220 * @new_mtu: new value for maximum frame size
4221 *
4222 * Returns 0 on success, negative on failure
4223 **/
4224 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4225 {
4226 struct igb_adapter *adapter = netdev_priv(netdev);
4227 struct pci_dev *pdev = adapter->pdev;
4228 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4229 u32 rx_buffer_len, i;
4230
4231 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4232 dev_err(&pdev->dev, "Invalid MTU setting\n");
4233 return -EINVAL;
4234 }
4235
4236 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4237 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4238 return -EINVAL;
4239 }
4240
4241 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4242 msleep(1);
4243
4244 /* igb_down has a dependency on max_frame_size */
4245 adapter->max_frame_size = max_frame;
4246
4247 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4248 * means we reserve 2 more, this pushes us to allocate from the next
4249 * larger slab size.
4250 * i.e. RXBUFFER_2048 --> size-4096 slab
4251 */
4252
4253 if (adapter->hw.mac.type == e1000_82580)
4254 max_frame += IGB_TS_HDR_LEN;
4255
4256 if (max_frame <= IGB_RXBUFFER_1024)
4257 rx_buffer_len = IGB_RXBUFFER_1024;
4258 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4259 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
4260 else
4261 rx_buffer_len = IGB_RXBUFFER_128;
4262
4263 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4264 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4265 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4266
4267 if ((adapter->hw.mac.type == e1000_82580) &&
4268 (rx_buffer_len == IGB_RXBUFFER_128))
4269 rx_buffer_len += IGB_RXBUFFER_64;
4270
4271 if (netif_running(netdev))
4272 igb_down(adapter);
4273
4274 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4275 netdev->mtu, new_mtu);
4276 netdev->mtu = new_mtu;
4277
4278 for (i = 0; i < adapter->num_rx_queues; i++)
4279 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4280
4281 if (netif_running(netdev))
4282 igb_up(adapter);
4283 else
4284 igb_reset(adapter);
4285
4286 clear_bit(__IGB_RESETTING, &adapter->state);
4287
4288 return 0;
4289 }
4290
4291 /**
4292 * igb_update_stats - Update the board statistics counters
4293 * @adapter: board private structure
4294 **/
4295
4296 void igb_update_stats(struct igb_adapter *adapter)
4297 {
4298 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
4299 struct e1000_hw *hw = &adapter->hw;
4300 struct pci_dev *pdev = adapter->pdev;
4301 u32 reg, mpc;
4302 u16 phy_tmp;
4303 int i;
4304 u64 bytes, packets;
4305
4306 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4307
4308 /*
4309 * Prevent stats update while adapter is being reset, or if the pci
4310 * connection is down.
4311 */
4312 if (adapter->link_speed == 0)
4313 return;
4314 if (pci_channel_offline(pdev))
4315 return;
4316
4317 bytes = 0;
4318 packets = 0;
4319 for (i = 0; i < adapter->num_rx_queues; i++) {
4320 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4321 struct igb_ring *ring = adapter->rx_ring[i];
4322 ring->rx_stats.drops += rqdpc_tmp;
4323 net_stats->rx_fifo_errors += rqdpc_tmp;
4324 bytes += ring->rx_stats.bytes;
4325 packets += ring->rx_stats.packets;
4326 }
4327
4328 net_stats->rx_bytes = bytes;
4329 net_stats->rx_packets = packets;
4330
4331 bytes = 0;
4332 packets = 0;
4333 for (i = 0; i < adapter->num_tx_queues; i++) {
4334 struct igb_ring *ring = adapter->tx_ring[i];
4335 bytes += ring->tx_stats.bytes;
4336 packets += ring->tx_stats.packets;
4337 }
4338 net_stats->tx_bytes = bytes;
4339 net_stats->tx_packets = packets;
4340
4341 /* read stats registers */
4342 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4343 adapter->stats.gprc += rd32(E1000_GPRC);
4344 adapter->stats.gorc += rd32(E1000_GORCL);
4345 rd32(E1000_GORCH); /* clear GORCL */
4346 adapter->stats.bprc += rd32(E1000_BPRC);
4347 adapter->stats.mprc += rd32(E1000_MPRC);
4348 adapter->stats.roc += rd32(E1000_ROC);
4349
4350 adapter->stats.prc64 += rd32(E1000_PRC64);
4351 adapter->stats.prc127 += rd32(E1000_PRC127);
4352 adapter->stats.prc255 += rd32(E1000_PRC255);
4353 adapter->stats.prc511 += rd32(E1000_PRC511);
4354 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4355 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4356 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4357 adapter->stats.sec += rd32(E1000_SEC);
4358
4359 mpc = rd32(E1000_MPC);
4360 adapter->stats.mpc += mpc;
4361 net_stats->rx_fifo_errors += mpc;
4362 adapter->stats.scc += rd32(E1000_SCC);
4363 adapter->stats.ecol += rd32(E1000_ECOL);
4364 adapter->stats.mcc += rd32(E1000_MCC);
4365 adapter->stats.latecol += rd32(E1000_LATECOL);
4366 adapter->stats.dc += rd32(E1000_DC);
4367 adapter->stats.rlec += rd32(E1000_RLEC);
4368 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4369 adapter->stats.xontxc += rd32(E1000_XONTXC);
4370 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4371 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4372 adapter->stats.fcruc += rd32(E1000_FCRUC);
4373 adapter->stats.gptc += rd32(E1000_GPTC);
4374 adapter->stats.gotc += rd32(E1000_GOTCL);
4375 rd32(E1000_GOTCH); /* clear GOTCL */
4376 adapter->stats.rnbc += rd32(E1000_RNBC);
4377 adapter->stats.ruc += rd32(E1000_RUC);
4378 adapter->stats.rfc += rd32(E1000_RFC);
4379 adapter->stats.rjc += rd32(E1000_RJC);
4380 adapter->stats.tor += rd32(E1000_TORH);
4381 adapter->stats.tot += rd32(E1000_TOTH);
4382 adapter->stats.tpr += rd32(E1000_TPR);
4383
4384 adapter->stats.ptc64 += rd32(E1000_PTC64);
4385 adapter->stats.ptc127 += rd32(E1000_PTC127);
4386 adapter->stats.ptc255 += rd32(E1000_PTC255);
4387 adapter->stats.ptc511 += rd32(E1000_PTC511);
4388 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4389 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4390
4391 adapter->stats.mptc += rd32(E1000_MPTC);
4392 adapter->stats.bptc += rd32(E1000_BPTC);
4393
4394 adapter->stats.tpt += rd32(E1000_TPT);
4395 adapter->stats.colc += rd32(E1000_COLC);
4396
4397 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4398 /* read internal phy specific stats */
4399 reg = rd32(E1000_CTRL_EXT);
4400 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4401 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4402 adapter->stats.tncrs += rd32(E1000_TNCRS);
4403 }
4404
4405 adapter->stats.tsctc += rd32(E1000_TSCTC);
4406 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4407
4408 adapter->stats.iac += rd32(E1000_IAC);
4409 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4410 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4411 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4412 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4413 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4414 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4415 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4416 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4417
4418 /* Fill out the OS statistics structure */
4419 net_stats->multicast = adapter->stats.mprc;
4420 net_stats->collisions = adapter->stats.colc;
4421
4422 /* Rx Errors */
4423
4424 /* RLEC on some newer hardware can be incorrect so build
4425 * our own version based on RUC and ROC */
4426 net_stats->rx_errors = adapter->stats.rxerrc +
4427 adapter->stats.crcerrs + adapter->stats.algnerrc +
4428 adapter->stats.ruc + adapter->stats.roc +
4429 adapter->stats.cexterr;
4430 net_stats->rx_length_errors = adapter->stats.ruc +
4431 adapter->stats.roc;
4432 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4433 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4434 net_stats->rx_missed_errors = adapter->stats.mpc;
4435
4436 /* Tx Errors */
4437 net_stats->tx_errors = adapter->stats.ecol +
4438 adapter->stats.latecol;
4439 net_stats->tx_aborted_errors = adapter->stats.ecol;
4440 net_stats->tx_window_errors = adapter->stats.latecol;
4441 net_stats->tx_carrier_errors = adapter->stats.tncrs;
4442
4443 /* Tx Dropped needs to be maintained elsewhere */
4444
4445 /* Phy Stats */
4446 if (hw->phy.media_type == e1000_media_type_copper) {
4447 if ((adapter->link_speed == SPEED_1000) &&
4448 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4449 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4450 adapter->phy_stats.idle_errors += phy_tmp;
4451 }
4452 }
4453
4454 /* Management Stats */
4455 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4456 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4457 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4458 }
4459
4460 static irqreturn_t igb_msix_other(int irq, void *data)
4461 {
4462 struct igb_adapter *adapter = data;
4463 struct e1000_hw *hw = &adapter->hw;
4464 u32 icr = rd32(E1000_ICR);
4465 /* reading ICR causes bit 31 of EICR to be cleared */
4466
4467 if (icr & E1000_ICR_DRSTA)
4468 schedule_work(&adapter->reset_task);
4469
4470 if (icr & E1000_ICR_DOUTSYNC) {
4471 /* HW is reporting DMA is out of sync */
4472 adapter->stats.doosync++;
4473 }
4474
4475 /* Check for a mailbox event */
4476 if (icr & E1000_ICR_VMMB)
4477 igb_msg_task(adapter);
4478
4479 if (icr & E1000_ICR_LSC) {
4480 hw->mac.get_link_status = 1;
4481 /* guard against interrupt when we're going down */
4482 if (!test_bit(__IGB_DOWN, &adapter->state))
4483 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4484 }
4485
4486 if (adapter->vfs_allocated_count)
4487 wr32(E1000_IMS, E1000_IMS_LSC |
4488 E1000_IMS_VMMB |
4489 E1000_IMS_DOUTSYNC);
4490 else
4491 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4492 wr32(E1000_EIMS, adapter->eims_other);
4493
4494 return IRQ_HANDLED;
4495 }
4496
4497 static void igb_write_itr(struct igb_q_vector *q_vector)
4498 {
4499 struct igb_adapter *adapter = q_vector->adapter;
4500 u32 itr_val = q_vector->itr_val & 0x7FFC;
4501
4502 if (!q_vector->set_itr)
4503 return;
4504
4505 if (!itr_val)
4506 itr_val = 0x4;
4507
4508 if (adapter->hw.mac.type == e1000_82575)
4509 itr_val |= itr_val << 16;
4510 else
4511 itr_val |= 0x8000000;
4512
4513 writel(itr_val, q_vector->itr_register);
4514 q_vector->set_itr = 0;
4515 }
4516
4517 static irqreturn_t igb_msix_ring(int irq, void *data)
4518 {
4519 struct igb_q_vector *q_vector = data;
4520
4521 /* Write the ITR value calculated from the previous interrupt. */
4522 igb_write_itr(q_vector);
4523
4524 napi_schedule(&q_vector->napi);
4525
4526 return IRQ_HANDLED;
4527 }
4528
4529 #ifdef CONFIG_IGB_DCA
4530 static void igb_update_dca(struct igb_q_vector *q_vector)
4531 {
4532 struct igb_adapter *adapter = q_vector->adapter;
4533 struct e1000_hw *hw = &adapter->hw;
4534 int cpu = get_cpu();
4535
4536 if (q_vector->cpu == cpu)
4537 goto out_no_update;
4538
4539 if (q_vector->tx_ring) {
4540 int q = q_vector->tx_ring->reg_idx;
4541 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4542 if (hw->mac.type == e1000_82575) {
4543 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4544 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4545 } else {
4546 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4547 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4548 E1000_DCA_TXCTRL_CPUID_SHIFT;
4549 }
4550 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4551 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4552 }
4553 if (q_vector->rx_ring) {
4554 int q = q_vector->rx_ring->reg_idx;
4555 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4556 if (hw->mac.type == e1000_82575) {
4557 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4558 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4559 } else {
4560 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4561 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4562 E1000_DCA_RXCTRL_CPUID_SHIFT;
4563 }
4564 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4565 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4566 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4567 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4568 }
4569 q_vector->cpu = cpu;
4570 out_no_update:
4571 put_cpu();
4572 }
4573
4574 static void igb_setup_dca(struct igb_adapter *adapter)
4575 {
4576 struct e1000_hw *hw = &adapter->hw;
4577 int i;
4578
4579 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4580 return;
4581
4582 /* Always use CB2 mode, difference is masked in the CB driver. */
4583 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4584
4585 for (i = 0; i < adapter->num_q_vectors; i++) {
4586 adapter->q_vector[i]->cpu = -1;
4587 igb_update_dca(adapter->q_vector[i]);
4588 }
4589 }
4590
4591 static int __igb_notify_dca(struct device *dev, void *data)
4592 {
4593 struct net_device *netdev = dev_get_drvdata(dev);
4594 struct igb_adapter *adapter = netdev_priv(netdev);
4595 struct pci_dev *pdev = adapter->pdev;
4596 struct e1000_hw *hw = &adapter->hw;
4597 unsigned long event = *(unsigned long *)data;
4598
4599 switch (event) {
4600 case DCA_PROVIDER_ADD:
4601 /* if already enabled, don't do it again */
4602 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4603 break;
4604 if (dca_add_requester(dev) == 0) {
4605 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4606 dev_info(&pdev->dev, "DCA enabled\n");
4607 igb_setup_dca(adapter);
4608 break;
4609 }
4610 /* Fall Through since DCA is disabled. */
4611 case DCA_PROVIDER_REMOVE:
4612 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4613 /* without this a class_device is left
4614 * hanging around in the sysfs model */
4615 dca_remove_requester(dev);
4616 dev_info(&pdev->dev, "DCA disabled\n");
4617 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4618 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4619 }
4620 break;
4621 }
4622
4623 return 0;
4624 }
4625
4626 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4627 void *p)
4628 {
4629 int ret_val;
4630
4631 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4632 __igb_notify_dca);
4633
4634 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4635 }
4636 #endif /* CONFIG_IGB_DCA */
4637
4638 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4639 {
4640 struct e1000_hw *hw = &adapter->hw;
4641 u32 ping;
4642 int i;
4643
4644 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4645 ping = E1000_PF_CONTROL_MSG;
4646 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4647 ping |= E1000_VT_MSGTYPE_CTS;
4648 igb_write_mbx(hw, &ping, 1, i);
4649 }
4650 }
4651
4652 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4653 {
4654 struct e1000_hw *hw = &adapter->hw;
4655 u32 vmolr = rd32(E1000_VMOLR(vf));
4656 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4657
4658 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4659 IGB_VF_FLAG_MULTI_PROMISC);
4660 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4661
4662 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4663 vmolr |= E1000_VMOLR_MPME;
4664 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4665 } else {
4666 /*
4667 * if we have hashes and we are clearing a multicast promisc
4668 * flag we need to write the hashes to the MTA as this step
4669 * was previously skipped
4670 */
4671 if (vf_data->num_vf_mc_hashes > 30) {
4672 vmolr |= E1000_VMOLR_MPME;
4673 } else if (vf_data->num_vf_mc_hashes) {
4674 int j;
4675 vmolr |= E1000_VMOLR_ROMPE;
4676 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4677 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4678 }
4679 }
4680
4681 wr32(E1000_VMOLR(vf), vmolr);
4682
4683 /* there are flags left unprocessed, likely not supported */
4684 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4685 return -EINVAL;
4686
4687 return 0;
4688
4689 }
4690
4691 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4692 u32 *msgbuf, u32 vf)
4693 {
4694 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4695 u16 *hash_list = (u16 *)&msgbuf[1];
4696 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4697 int i;
4698
4699 /* salt away the number of multicast addresses assigned
4700 * to this VF for later use to restore when the PF multi cast
4701 * list changes
4702 */
4703 vf_data->num_vf_mc_hashes = n;
4704
4705 /* only up to 30 hash values supported */
4706 if (n > 30)
4707 n = 30;
4708
4709 /* store the hashes for later use */
4710 for (i = 0; i < n; i++)
4711 vf_data->vf_mc_hashes[i] = hash_list[i];
4712
4713 /* Flush and reset the mta with the new values */
4714 igb_set_rx_mode(adapter->netdev);
4715
4716 return 0;
4717 }
4718
4719 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4720 {
4721 struct e1000_hw *hw = &adapter->hw;
4722 struct vf_data_storage *vf_data;
4723 int i, j;
4724
4725 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4726 u32 vmolr = rd32(E1000_VMOLR(i));
4727 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4728
4729 vf_data = &adapter->vf_data[i];
4730
4731 if ((vf_data->num_vf_mc_hashes > 30) ||
4732 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4733 vmolr |= E1000_VMOLR_MPME;
4734 } else if (vf_data->num_vf_mc_hashes) {
4735 vmolr |= E1000_VMOLR_ROMPE;
4736 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4737 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4738 }
4739 wr32(E1000_VMOLR(i), vmolr);
4740 }
4741 }
4742
4743 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4744 {
4745 struct e1000_hw *hw = &adapter->hw;
4746 u32 pool_mask, reg, vid;
4747 int i;
4748
4749 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4750
4751 /* Find the vlan filter for this id */
4752 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4753 reg = rd32(E1000_VLVF(i));
4754
4755 /* remove the vf from the pool */
4756 reg &= ~pool_mask;
4757
4758 /* if pool is empty then remove entry from vfta */
4759 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4760 (reg & E1000_VLVF_VLANID_ENABLE)) {
4761 reg = 0;
4762 vid = reg & E1000_VLVF_VLANID_MASK;
4763 igb_vfta_set(hw, vid, false);
4764 }
4765
4766 wr32(E1000_VLVF(i), reg);
4767 }
4768
4769 adapter->vf_data[vf].vlans_enabled = 0;
4770 }
4771
4772 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4773 {
4774 struct e1000_hw *hw = &adapter->hw;
4775 u32 reg, i;
4776
4777 /* The vlvf table only exists on 82576 hardware and newer */
4778 if (hw->mac.type < e1000_82576)
4779 return -1;
4780
4781 /* we only need to do this if VMDq is enabled */
4782 if (!adapter->vfs_allocated_count)
4783 return -1;
4784
4785 /* Find the vlan filter for this id */
4786 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4787 reg = rd32(E1000_VLVF(i));
4788 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4789 vid == (reg & E1000_VLVF_VLANID_MASK))
4790 break;
4791 }
4792
4793 if (add) {
4794 if (i == E1000_VLVF_ARRAY_SIZE) {
4795 /* Did not find a matching VLAN ID entry that was
4796 * enabled. Search for a free filter entry, i.e.
4797 * one without the enable bit set
4798 */
4799 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4800 reg = rd32(E1000_VLVF(i));
4801 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4802 break;
4803 }
4804 }
4805 if (i < E1000_VLVF_ARRAY_SIZE) {
4806 /* Found an enabled/available entry */
4807 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4808
4809 /* if !enabled we need to set this up in vfta */
4810 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4811 /* add VID to filter table */
4812 igb_vfta_set(hw, vid, true);
4813 reg |= E1000_VLVF_VLANID_ENABLE;
4814 }
4815 reg &= ~E1000_VLVF_VLANID_MASK;
4816 reg |= vid;
4817 wr32(E1000_VLVF(i), reg);
4818
4819 /* do not modify RLPML for PF devices */
4820 if (vf >= adapter->vfs_allocated_count)
4821 return 0;
4822
4823 if (!adapter->vf_data[vf].vlans_enabled) {
4824 u32 size;
4825 reg = rd32(E1000_VMOLR(vf));
4826 size = reg & E1000_VMOLR_RLPML_MASK;
4827 size += 4;
4828 reg &= ~E1000_VMOLR_RLPML_MASK;
4829 reg |= size;
4830 wr32(E1000_VMOLR(vf), reg);
4831 }
4832
4833 adapter->vf_data[vf].vlans_enabled++;
4834 return 0;
4835 }
4836 } else {
4837 if (i < E1000_VLVF_ARRAY_SIZE) {
4838 /* remove vf from the pool */
4839 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4840 /* if pool is empty then remove entry from vfta */
4841 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4842 reg = 0;
4843 igb_vfta_set(hw, vid, false);
4844 }
4845 wr32(E1000_VLVF(i), reg);
4846
4847 /* do not modify RLPML for PF devices */
4848 if (vf >= adapter->vfs_allocated_count)
4849 return 0;
4850
4851 adapter->vf_data[vf].vlans_enabled--;
4852 if (!adapter->vf_data[vf].vlans_enabled) {
4853 u32 size;
4854 reg = rd32(E1000_VMOLR(vf));
4855 size = reg & E1000_VMOLR_RLPML_MASK;
4856 size -= 4;
4857 reg &= ~E1000_VMOLR_RLPML_MASK;
4858 reg |= size;
4859 wr32(E1000_VMOLR(vf), reg);
4860 }
4861 }
4862 }
4863 return 0;
4864 }
4865
4866 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4867 {
4868 struct e1000_hw *hw = &adapter->hw;
4869
4870 if (vid)
4871 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4872 else
4873 wr32(E1000_VMVIR(vf), 0);
4874 }
4875
4876 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4877 int vf, u16 vlan, u8 qos)
4878 {
4879 int err = 0;
4880 struct igb_adapter *adapter = netdev_priv(netdev);
4881
4882 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4883 return -EINVAL;
4884 if (vlan || qos) {
4885 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4886 if (err)
4887 goto out;
4888 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4889 igb_set_vmolr(adapter, vf, !vlan);
4890 adapter->vf_data[vf].pf_vlan = vlan;
4891 adapter->vf_data[vf].pf_qos = qos;
4892 dev_info(&adapter->pdev->dev,
4893 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4894 if (test_bit(__IGB_DOWN, &adapter->state)) {
4895 dev_warn(&adapter->pdev->dev,
4896 "The VF VLAN has been set,"
4897 " but the PF device is not up.\n");
4898 dev_warn(&adapter->pdev->dev,
4899 "Bring the PF device up before"
4900 " attempting to use the VF device.\n");
4901 }
4902 } else {
4903 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4904 false, vf);
4905 igb_set_vmvir(adapter, vlan, vf);
4906 igb_set_vmolr(adapter, vf, true);
4907 adapter->vf_data[vf].pf_vlan = 0;
4908 adapter->vf_data[vf].pf_qos = 0;
4909 }
4910 out:
4911 return err;
4912 }
4913
4914 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4915 {
4916 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4917 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4918
4919 return igb_vlvf_set(adapter, vid, add, vf);
4920 }
4921
4922 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4923 {
4924 /* clear flags */
4925 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
4926 adapter->vf_data[vf].last_nack = jiffies;
4927
4928 /* reset offloads to defaults */
4929 igb_set_vmolr(adapter, vf, true);
4930
4931 /* reset vlans for device */
4932 igb_clear_vf_vfta(adapter, vf);
4933 if (adapter->vf_data[vf].pf_vlan)
4934 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4935 adapter->vf_data[vf].pf_vlan,
4936 adapter->vf_data[vf].pf_qos);
4937 else
4938 igb_clear_vf_vfta(adapter, vf);
4939
4940 /* reset multicast table array for vf */
4941 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4942
4943 /* Flush and reset the mta with the new values */
4944 igb_set_rx_mode(adapter->netdev);
4945 }
4946
4947 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4948 {
4949 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4950
4951 /* generate a new mac address as we were hotplug removed/added */
4952 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4953 random_ether_addr(vf_mac);
4954
4955 /* process remaining reset events */
4956 igb_vf_reset(adapter, vf);
4957 }
4958
4959 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4960 {
4961 struct e1000_hw *hw = &adapter->hw;
4962 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4963 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4964 u32 reg, msgbuf[3];
4965 u8 *addr = (u8 *)(&msgbuf[1]);
4966
4967 /* process all the same items cleared in a function level reset */
4968 igb_vf_reset(adapter, vf);
4969
4970 /* set vf mac address */
4971 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4972
4973 /* enable transmit and receive for vf */
4974 reg = rd32(E1000_VFTE);
4975 wr32(E1000_VFTE, reg | (1 << vf));
4976 reg = rd32(E1000_VFRE);
4977 wr32(E1000_VFRE, reg | (1 << vf));
4978
4979 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4980
4981 /* reply to reset with ack and vf mac address */
4982 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4983 memcpy(addr, vf_mac, 6);
4984 igb_write_mbx(hw, msgbuf, 3, vf);
4985 }
4986
4987 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4988 {
4989 unsigned char *addr = (char *)&msg[1];
4990 int err = -1;
4991
4992 if (is_valid_ether_addr(addr))
4993 err = igb_set_vf_mac(adapter, vf, addr);
4994
4995 return err;
4996 }
4997
4998 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4999 {
5000 struct e1000_hw *hw = &adapter->hw;
5001 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5002 u32 msg = E1000_VT_MSGTYPE_NACK;
5003
5004 /* if device isn't clear to send it shouldn't be reading either */
5005 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5006 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5007 igb_write_mbx(hw, &msg, 1, vf);
5008 vf_data->last_nack = jiffies;
5009 }
5010 }
5011
5012 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5013 {
5014 struct pci_dev *pdev = adapter->pdev;
5015 u32 msgbuf[E1000_VFMAILBOX_SIZE];
5016 struct e1000_hw *hw = &adapter->hw;
5017 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5018 s32 retval;
5019
5020 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5021
5022 if (retval) {
5023 /* if receive failed revoke VF CTS stats and restart init */
5024 dev_err(&pdev->dev, "Error receiving message from VF\n");
5025 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5026 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5027 return;
5028 goto out;
5029 }
5030
5031 /* this is a message we already processed, do nothing */
5032 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5033 return;
5034
5035 /*
5036 * until the vf completes a reset it should not be
5037 * allowed to start any configuration.
5038 */
5039
5040 if (msgbuf[0] == E1000_VF_RESET) {
5041 igb_vf_reset_msg(adapter, vf);
5042 return;
5043 }
5044
5045 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5046 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5047 return;
5048 retval = -1;
5049 goto out;
5050 }
5051
5052 switch ((msgbuf[0] & 0xFFFF)) {
5053 case E1000_VF_SET_MAC_ADDR:
5054 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5055 break;
5056 case E1000_VF_SET_PROMISC:
5057 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5058 break;
5059 case E1000_VF_SET_MULTICAST:
5060 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5061 break;
5062 case E1000_VF_SET_LPE:
5063 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5064 break;
5065 case E1000_VF_SET_VLAN:
5066 if (adapter->vf_data[vf].pf_vlan)
5067 retval = -1;
5068 else
5069 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5070 break;
5071 default:
5072 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5073 retval = -1;
5074 break;
5075 }
5076
5077 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5078 out:
5079 /* notify the VF of the results of what it sent us */
5080 if (retval)
5081 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5082 else
5083 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5084
5085 igb_write_mbx(hw, msgbuf, 1, vf);
5086 }
5087
5088 static void igb_msg_task(struct igb_adapter *adapter)
5089 {
5090 struct e1000_hw *hw = &adapter->hw;
5091 u32 vf;
5092
5093 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5094 /* process any reset requests */
5095 if (!igb_check_for_rst(hw, vf))
5096 igb_vf_reset_event(adapter, vf);
5097
5098 /* process any messages pending */
5099 if (!igb_check_for_msg(hw, vf))
5100 igb_rcv_msg_from_vf(adapter, vf);
5101
5102 /* process any acks */
5103 if (!igb_check_for_ack(hw, vf))
5104 igb_rcv_ack_from_vf(adapter, vf);
5105 }
5106 }
5107
5108 /**
5109 * igb_set_uta - Set unicast filter table address
5110 * @adapter: board private structure
5111 *
5112 * The unicast table address is a register array of 32-bit registers.
5113 * The table is meant to be used in a way similar to how the MTA is used
5114 * however due to certain limitations in the hardware it is necessary to
5115 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5116 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5117 **/
5118 static void igb_set_uta(struct igb_adapter *adapter)
5119 {
5120 struct e1000_hw *hw = &adapter->hw;
5121 int i;
5122
5123 /* The UTA table only exists on 82576 hardware and newer */
5124 if (hw->mac.type < e1000_82576)
5125 return;
5126
5127 /* we only need to do this if VMDq is enabled */
5128 if (!adapter->vfs_allocated_count)
5129 return;
5130
5131 for (i = 0; i < hw->mac.uta_reg_count; i++)
5132 array_wr32(E1000_UTA, i, ~0);
5133 }
5134
5135 /**
5136 * igb_intr_msi - Interrupt Handler
5137 * @irq: interrupt number
5138 * @data: pointer to a network interface device structure
5139 **/
5140 static irqreturn_t igb_intr_msi(int irq, void *data)
5141 {
5142 struct igb_adapter *adapter = data;
5143 struct igb_q_vector *q_vector = adapter->q_vector[0];
5144 struct e1000_hw *hw = &adapter->hw;
5145 /* read ICR disables interrupts using IAM */
5146 u32 icr = rd32(E1000_ICR);
5147
5148 igb_write_itr(q_vector);
5149
5150 if (icr & E1000_ICR_DRSTA)
5151 schedule_work(&adapter->reset_task);
5152
5153 if (icr & E1000_ICR_DOUTSYNC) {
5154 /* HW is reporting DMA is out of sync */
5155 adapter->stats.doosync++;
5156 }
5157
5158 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5159 hw->mac.get_link_status = 1;
5160 if (!test_bit(__IGB_DOWN, &adapter->state))
5161 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5162 }
5163
5164 napi_schedule(&q_vector->napi);
5165
5166 return IRQ_HANDLED;
5167 }
5168
5169 /**
5170 * igb_intr - Legacy Interrupt Handler
5171 * @irq: interrupt number
5172 * @data: pointer to a network interface device structure
5173 **/
5174 static irqreturn_t igb_intr(int irq, void *data)
5175 {
5176 struct igb_adapter *adapter = data;
5177 struct igb_q_vector *q_vector = adapter->q_vector[0];
5178 struct e1000_hw *hw = &adapter->hw;
5179 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5180 * need for the IMC write */
5181 u32 icr = rd32(E1000_ICR);
5182 if (!icr)
5183 return IRQ_NONE; /* Not our interrupt */
5184
5185 igb_write_itr(q_vector);
5186
5187 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5188 * not set, then the adapter didn't send an interrupt */
5189 if (!(icr & E1000_ICR_INT_ASSERTED))
5190 return IRQ_NONE;
5191
5192 if (icr & E1000_ICR_DRSTA)
5193 schedule_work(&adapter->reset_task);
5194
5195 if (icr & E1000_ICR_DOUTSYNC) {
5196 /* HW is reporting DMA is out of sync */
5197 adapter->stats.doosync++;
5198 }
5199
5200 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5201 hw->mac.get_link_status = 1;
5202 /* guard against interrupt when we're going down */
5203 if (!test_bit(__IGB_DOWN, &adapter->state))
5204 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5205 }
5206
5207 napi_schedule(&q_vector->napi);
5208
5209 return IRQ_HANDLED;
5210 }
5211
5212 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5213 {
5214 struct igb_adapter *adapter = q_vector->adapter;
5215 struct e1000_hw *hw = &adapter->hw;
5216
5217 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5218 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
5219 if (!adapter->msix_entries)
5220 igb_set_itr(adapter);
5221 else
5222 igb_update_ring_itr(q_vector);
5223 }
5224
5225 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5226 if (adapter->msix_entries)
5227 wr32(E1000_EIMS, q_vector->eims_value);
5228 else
5229 igb_irq_enable(adapter);
5230 }
5231 }
5232
5233 /**
5234 * igb_poll - NAPI Rx polling callback
5235 * @napi: napi polling structure
5236 * @budget: count of how many packets we should handle
5237 **/
5238 static int igb_poll(struct napi_struct *napi, int budget)
5239 {
5240 struct igb_q_vector *q_vector = container_of(napi,
5241 struct igb_q_vector,
5242 napi);
5243 int tx_clean_complete = 1, work_done = 0;
5244
5245 #ifdef CONFIG_IGB_DCA
5246 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5247 igb_update_dca(q_vector);
5248 #endif
5249 if (q_vector->tx_ring)
5250 tx_clean_complete = igb_clean_tx_irq(q_vector);
5251
5252 if (q_vector->rx_ring)
5253 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5254
5255 if (!tx_clean_complete)
5256 work_done = budget;
5257
5258 /* If not enough Rx work done, exit the polling mode */
5259 if (work_done < budget) {
5260 napi_complete(napi);
5261 igb_ring_irq_enable(q_vector);
5262 }
5263
5264 return work_done;
5265 }
5266
5267 /**
5268 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
5269 * @adapter: board private structure
5270 * @shhwtstamps: timestamp structure to update
5271 * @regval: unsigned 64bit system time value.
5272 *
5273 * We need to convert the system time value stored in the RX/TXSTMP registers
5274 * into a hwtstamp which can be used by the upper level timestamping functions
5275 */
5276 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5277 struct skb_shared_hwtstamps *shhwtstamps,
5278 u64 regval)
5279 {
5280 u64 ns;
5281
5282 /*
5283 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5284 * 24 to match clock shift we setup earlier.
5285 */
5286 if (adapter->hw.mac.type == e1000_82580)
5287 regval <<= IGB_82580_TSYNC_SHIFT;
5288
5289 ns = timecounter_cyc2time(&adapter->clock, regval);
5290 timecompare_update(&adapter->compare, ns);
5291 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5292 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5293 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5294 }
5295
5296 /**
5297 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5298 * @q_vector: pointer to q_vector containing needed info
5299 * @buffer: pointer to igb_buffer structure
5300 *
5301 * If we were asked to do hardware stamping and such a time stamp is
5302 * available, then it must have been for this skb here because we only
5303 * allow only one such packet into the queue.
5304 */
5305 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
5306 {
5307 struct igb_adapter *adapter = q_vector->adapter;
5308 struct e1000_hw *hw = &adapter->hw;
5309 struct skb_shared_hwtstamps shhwtstamps;
5310 u64 regval;
5311
5312 /* if skb does not support hw timestamp or TX stamp not valid exit */
5313 if (likely(!buffer_info->shtx.hardware) ||
5314 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5315 return;
5316
5317 regval = rd32(E1000_TXSTMPL);
5318 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5319
5320 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5321 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5322 }
5323
5324 /**
5325 * igb_clean_tx_irq - Reclaim resources after transmit completes
5326 * @q_vector: pointer to q_vector containing needed info
5327 * returns true if ring is completely cleaned
5328 **/
5329 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5330 {
5331 struct igb_adapter *adapter = q_vector->adapter;
5332 struct igb_ring *tx_ring = q_vector->tx_ring;
5333 struct net_device *netdev = tx_ring->netdev;
5334 struct e1000_hw *hw = &adapter->hw;
5335 struct igb_buffer *buffer_info;
5336 union e1000_adv_tx_desc *tx_desc, *eop_desc;
5337 unsigned int total_bytes = 0, total_packets = 0;
5338 unsigned int i, eop, count = 0;
5339 bool cleaned = false;
5340
5341 i = tx_ring->next_to_clean;
5342 eop = tx_ring->buffer_info[i].next_to_watch;
5343 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5344
5345 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5346 (count < tx_ring->count)) {
5347 for (cleaned = false; !cleaned; count++) {
5348 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
5349 buffer_info = &tx_ring->buffer_info[i];
5350 cleaned = (i == eop);
5351
5352 if (buffer_info->skb) {
5353 total_bytes += buffer_info->bytecount;
5354 /* gso_segs is currently only valid for tcp */
5355 total_packets += buffer_info->gso_segs;
5356 igb_tx_hwtstamp(q_vector, buffer_info);
5357 }
5358
5359 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
5360 tx_desc->wb.status = 0;
5361
5362 i++;
5363 if (i == tx_ring->count)
5364 i = 0;
5365 }
5366 eop = tx_ring->buffer_info[i].next_to_watch;
5367 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5368 }
5369
5370 tx_ring->next_to_clean = i;
5371
5372 if (unlikely(count &&
5373 netif_carrier_ok(netdev) &&
5374 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5375 /* Make sure that anybody stopping the queue after this
5376 * sees the new next_to_clean.
5377 */
5378 smp_mb();
5379 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5380 !(test_bit(__IGB_DOWN, &adapter->state))) {
5381 netif_wake_subqueue(netdev, tx_ring->queue_index);
5382 tx_ring->tx_stats.restart_queue++;
5383 }
5384 }
5385
5386 if (tx_ring->detect_tx_hung) {
5387 /* Detect a transmit hang in hardware, this serializes the
5388 * check with the clearing of time_stamp and movement of i */
5389 tx_ring->detect_tx_hung = false;
5390 if (tx_ring->buffer_info[i].time_stamp &&
5391 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
5392 (adapter->tx_timeout_factor * HZ)) &&
5393 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5394
5395 /* detected Tx unit hang */
5396 dev_err(tx_ring->dev,
5397 "Detected Tx Unit Hang\n"
5398 " Tx Queue <%d>\n"
5399 " TDH <%x>\n"
5400 " TDT <%x>\n"
5401 " next_to_use <%x>\n"
5402 " next_to_clean <%x>\n"
5403 "buffer_info[next_to_clean]\n"
5404 " time_stamp <%lx>\n"
5405 " next_to_watch <%x>\n"
5406 " jiffies <%lx>\n"
5407 " desc.status <%x>\n",
5408 tx_ring->queue_index,
5409 readl(tx_ring->head),
5410 readl(tx_ring->tail),
5411 tx_ring->next_to_use,
5412 tx_ring->next_to_clean,
5413 tx_ring->buffer_info[eop].time_stamp,
5414 eop,
5415 jiffies,
5416 eop_desc->wb.status);
5417 netif_stop_subqueue(netdev, tx_ring->queue_index);
5418 }
5419 }
5420 tx_ring->total_bytes += total_bytes;
5421 tx_ring->total_packets += total_packets;
5422 tx_ring->tx_stats.bytes += total_bytes;
5423 tx_ring->tx_stats.packets += total_packets;
5424 return (count < tx_ring->count);
5425 }
5426
5427 /**
5428 * igb_receive_skb - helper function to handle rx indications
5429 * @q_vector: structure containing interrupt and ring information
5430 * @skb: packet to send up
5431 * @vlan_tag: vlan tag for packet
5432 **/
5433 static void igb_receive_skb(struct igb_q_vector *q_vector,
5434 struct sk_buff *skb,
5435 u16 vlan_tag)
5436 {
5437 struct igb_adapter *adapter = q_vector->adapter;
5438
5439 if (vlan_tag && adapter->vlgrp)
5440 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5441 vlan_tag, skb);
5442 else
5443 napi_gro_receive(&q_vector->napi, skb);
5444 }
5445
5446 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5447 u32 status_err, struct sk_buff *skb)
5448 {
5449 skb->ip_summed = CHECKSUM_NONE;
5450
5451 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5452 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5453 (status_err & E1000_RXD_STAT_IXSM))
5454 return;
5455
5456 /* TCP/UDP checksum error bit is set */
5457 if (status_err &
5458 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5459 /*
5460 * work around errata with sctp packets where the TCPE aka
5461 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5462 * packets, (aka let the stack check the crc32c)
5463 */
5464 if ((skb->len == 60) &&
5465 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
5466 ring->rx_stats.csum_err++;
5467
5468 /* let the stack verify checksum errors */
5469 return;
5470 }
5471 /* It must be a TCP or UDP packet with a valid checksum */
5472 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5473 skb->ip_summed = CHECKSUM_UNNECESSARY;
5474
5475 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
5476 }
5477
5478 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5479 struct sk_buff *skb)
5480 {
5481 struct igb_adapter *adapter = q_vector->adapter;
5482 struct e1000_hw *hw = &adapter->hw;
5483 u64 regval;
5484
5485 /*
5486 * If this bit is set, then the RX registers contain the time stamp. No
5487 * other packet will be time stamped until we read these registers, so
5488 * read the registers to make them available again. Because only one
5489 * packet can be time stamped at a time, we know that the register
5490 * values must belong to this one here and therefore we don't need to
5491 * compare any of the additional attributes stored for it.
5492 *
5493 * If nothing went wrong, then it should have a skb_shared_tx that we
5494 * can turn into a skb_shared_hwtstamps.
5495 */
5496 if (staterr & E1000_RXDADV_STAT_TSIP) {
5497 u32 *stamp = (u32 *)skb->data;
5498 regval = le32_to_cpu(*(stamp + 2));
5499 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5500 skb_pull(skb, IGB_TS_HDR_LEN);
5501 } else {
5502 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5503 return;
5504
5505 regval = rd32(E1000_RXSTMPL);
5506 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5507 }
5508
5509 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5510 }
5511 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5512 union e1000_adv_rx_desc *rx_desc)
5513 {
5514 /* HW will not DMA in data larger than the given buffer, even if it
5515 * parses the (NFS, of course) header to be larger. In that case, it
5516 * fills the header buffer and spills the rest into the page.
5517 */
5518 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5519 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5520 if (hlen > rx_ring->rx_buffer_len)
5521 hlen = rx_ring->rx_buffer_len;
5522 return hlen;
5523 }
5524
5525 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5526 int *work_done, int budget)
5527 {
5528 struct igb_ring *rx_ring = q_vector->rx_ring;
5529 struct net_device *netdev = rx_ring->netdev;
5530 struct device *dev = rx_ring->dev;
5531 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5532 struct igb_buffer *buffer_info , *next_buffer;
5533 struct sk_buff *skb;
5534 bool cleaned = false;
5535 int cleaned_count = 0;
5536 int current_node = numa_node_id();
5537 unsigned int total_bytes = 0, total_packets = 0;
5538 unsigned int i;
5539 u32 staterr;
5540 u16 length;
5541 u16 vlan_tag;
5542
5543 i = rx_ring->next_to_clean;
5544 buffer_info = &rx_ring->buffer_info[i];
5545 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5546 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5547
5548 while (staterr & E1000_RXD_STAT_DD) {
5549 if (*work_done >= budget)
5550 break;
5551 (*work_done)++;
5552
5553 skb = buffer_info->skb;
5554 prefetch(skb->data - NET_IP_ALIGN);
5555 buffer_info->skb = NULL;
5556
5557 i++;
5558 if (i == rx_ring->count)
5559 i = 0;
5560
5561 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5562 prefetch(next_rxd);
5563 next_buffer = &rx_ring->buffer_info[i];
5564
5565 length = le16_to_cpu(rx_desc->wb.upper.length);
5566 cleaned = true;
5567 cleaned_count++;
5568
5569 if (buffer_info->dma) {
5570 dma_unmap_single(dev, buffer_info->dma,
5571 rx_ring->rx_buffer_len,
5572 DMA_FROM_DEVICE);
5573 buffer_info->dma = 0;
5574 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5575 skb_put(skb, length);
5576 goto send_up;
5577 }
5578 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5579 }
5580
5581 if (length) {
5582 dma_unmap_page(dev, buffer_info->page_dma,
5583 PAGE_SIZE / 2, DMA_FROM_DEVICE);
5584 buffer_info->page_dma = 0;
5585
5586 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5587 buffer_info->page,
5588 buffer_info->page_offset,
5589 length);
5590
5591 if ((page_count(buffer_info->page) != 1) ||
5592 (page_to_nid(buffer_info->page) != current_node))
5593 buffer_info->page = NULL;
5594 else
5595 get_page(buffer_info->page);
5596
5597 skb->len += length;
5598 skb->data_len += length;
5599 skb->truesize += length;
5600 }
5601
5602 if (!(staterr & E1000_RXD_STAT_EOP)) {
5603 buffer_info->skb = next_buffer->skb;
5604 buffer_info->dma = next_buffer->dma;
5605 next_buffer->skb = skb;
5606 next_buffer->dma = 0;
5607 goto next_desc;
5608 }
5609 send_up:
5610 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5611 dev_kfree_skb_irq(skb);
5612 goto next_desc;
5613 }
5614
5615 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5616 igb_rx_hwtstamp(q_vector, staterr, skb);
5617 total_bytes += skb->len;
5618 total_packets++;
5619
5620 igb_rx_checksum_adv(rx_ring, staterr, skb);
5621
5622 skb->protocol = eth_type_trans(skb, netdev);
5623 skb_record_rx_queue(skb, rx_ring->queue_index);
5624
5625 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5626 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5627
5628 igb_receive_skb(q_vector, skb, vlan_tag);
5629
5630 next_desc:
5631 rx_desc->wb.upper.status_error = 0;
5632
5633 /* return some buffers to hardware, one at a time is too slow */
5634 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5635 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5636 cleaned_count = 0;
5637 }
5638
5639 /* use prefetched values */
5640 rx_desc = next_rxd;
5641 buffer_info = next_buffer;
5642 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5643 }
5644
5645 rx_ring->next_to_clean = i;
5646 cleaned_count = igb_desc_unused(rx_ring);
5647
5648 if (cleaned_count)
5649 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5650
5651 rx_ring->total_packets += total_packets;
5652 rx_ring->total_bytes += total_bytes;
5653 rx_ring->rx_stats.packets += total_packets;
5654 rx_ring->rx_stats.bytes += total_bytes;
5655 return cleaned;
5656 }
5657
5658 /**
5659 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5660 * @adapter: address of board private structure
5661 **/
5662 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5663 {
5664 struct net_device *netdev = rx_ring->netdev;
5665 union e1000_adv_rx_desc *rx_desc;
5666 struct igb_buffer *buffer_info;
5667 struct sk_buff *skb;
5668 unsigned int i;
5669 int bufsz;
5670
5671 i = rx_ring->next_to_use;
5672 buffer_info = &rx_ring->buffer_info[i];
5673
5674 bufsz = rx_ring->rx_buffer_len;
5675
5676 while (cleaned_count--) {
5677 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5678
5679 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5680 if (!buffer_info->page) {
5681 buffer_info->page = netdev_alloc_page(netdev);
5682 if (!buffer_info->page) {
5683 rx_ring->rx_stats.alloc_failed++;
5684 goto no_buffers;
5685 }
5686 buffer_info->page_offset = 0;
5687 } else {
5688 buffer_info->page_offset ^= PAGE_SIZE / 2;
5689 }
5690 buffer_info->page_dma =
5691 dma_map_page(rx_ring->dev, buffer_info->page,
5692 buffer_info->page_offset,
5693 PAGE_SIZE / 2,
5694 DMA_FROM_DEVICE);
5695 if (dma_mapping_error(rx_ring->dev,
5696 buffer_info->page_dma)) {
5697 buffer_info->page_dma = 0;
5698 rx_ring->rx_stats.alloc_failed++;
5699 goto no_buffers;
5700 }
5701 }
5702
5703 skb = buffer_info->skb;
5704 if (!skb) {
5705 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5706 if (!skb) {
5707 rx_ring->rx_stats.alloc_failed++;
5708 goto no_buffers;
5709 }
5710
5711 buffer_info->skb = skb;
5712 }
5713 if (!buffer_info->dma) {
5714 buffer_info->dma = dma_map_single(rx_ring->dev,
5715 skb->data,
5716 bufsz,
5717 DMA_FROM_DEVICE);
5718 if (dma_mapping_error(rx_ring->dev,
5719 buffer_info->dma)) {
5720 buffer_info->dma = 0;
5721 rx_ring->rx_stats.alloc_failed++;
5722 goto no_buffers;
5723 }
5724 }
5725 /* Refresh the desc even if buffer_addrs didn't change because
5726 * each write-back erases this info. */
5727 if (bufsz < IGB_RXBUFFER_1024) {
5728 rx_desc->read.pkt_addr =
5729 cpu_to_le64(buffer_info->page_dma);
5730 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5731 } else {
5732 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5733 rx_desc->read.hdr_addr = 0;
5734 }
5735
5736 i++;
5737 if (i == rx_ring->count)
5738 i = 0;
5739 buffer_info = &rx_ring->buffer_info[i];
5740 }
5741
5742 no_buffers:
5743 if (rx_ring->next_to_use != i) {
5744 rx_ring->next_to_use = i;
5745 if (i == 0)
5746 i = (rx_ring->count - 1);
5747 else
5748 i--;
5749
5750 /* Force memory writes to complete before letting h/w
5751 * know there are new descriptors to fetch. (Only
5752 * applicable for weak-ordered memory model archs,
5753 * such as IA-64). */
5754 wmb();
5755 writel(i, rx_ring->tail);
5756 }
5757 }
5758
5759 /**
5760 * igb_mii_ioctl -
5761 * @netdev:
5762 * @ifreq:
5763 * @cmd:
5764 **/
5765 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5766 {
5767 struct igb_adapter *adapter = netdev_priv(netdev);
5768 struct mii_ioctl_data *data = if_mii(ifr);
5769
5770 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5771 return -EOPNOTSUPP;
5772
5773 switch (cmd) {
5774 case SIOCGMIIPHY:
5775 data->phy_id = adapter->hw.phy.addr;
5776 break;
5777 case SIOCGMIIREG:
5778 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5779 &data->val_out))
5780 return -EIO;
5781 break;
5782 case SIOCSMIIREG:
5783 default:
5784 return -EOPNOTSUPP;
5785 }
5786 return 0;
5787 }
5788
5789 /**
5790 * igb_hwtstamp_ioctl - control hardware time stamping
5791 * @netdev:
5792 * @ifreq:
5793 * @cmd:
5794 *
5795 * Outgoing time stamping can be enabled and disabled. Play nice and
5796 * disable it when requested, although it shouldn't case any overhead
5797 * when no packet needs it. At most one packet in the queue may be
5798 * marked for time stamping, otherwise it would be impossible to tell
5799 * for sure to which packet the hardware time stamp belongs.
5800 *
5801 * Incoming time stamping has to be configured via the hardware
5802 * filters. Not all combinations are supported, in particular event
5803 * type has to be specified. Matching the kind of event packet is
5804 * not supported, with the exception of "all V2 events regardless of
5805 * level 2 or 4".
5806 *
5807 **/
5808 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5809 struct ifreq *ifr, int cmd)
5810 {
5811 struct igb_adapter *adapter = netdev_priv(netdev);
5812 struct e1000_hw *hw = &adapter->hw;
5813 struct hwtstamp_config config;
5814 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5815 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5816 u32 tsync_rx_cfg = 0;
5817 bool is_l4 = false;
5818 bool is_l2 = false;
5819 u32 regval;
5820
5821 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5822 return -EFAULT;
5823
5824 /* reserved for future extensions */
5825 if (config.flags)
5826 return -EINVAL;
5827
5828 switch (config.tx_type) {
5829 case HWTSTAMP_TX_OFF:
5830 tsync_tx_ctl = 0;
5831 case HWTSTAMP_TX_ON:
5832 break;
5833 default:
5834 return -ERANGE;
5835 }
5836
5837 switch (config.rx_filter) {
5838 case HWTSTAMP_FILTER_NONE:
5839 tsync_rx_ctl = 0;
5840 break;
5841 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5842 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5843 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5844 case HWTSTAMP_FILTER_ALL:
5845 /*
5846 * register TSYNCRXCFG must be set, therefore it is not
5847 * possible to time stamp both Sync and Delay_Req messages
5848 * => fall back to time stamping all packets
5849 */
5850 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5851 config.rx_filter = HWTSTAMP_FILTER_ALL;
5852 break;
5853 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5854 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5855 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5856 is_l4 = true;
5857 break;
5858 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5859 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5860 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5861 is_l4 = true;
5862 break;
5863 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5864 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5865 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5866 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5867 is_l2 = true;
5868 is_l4 = true;
5869 config.rx_filter = HWTSTAMP_FILTER_SOME;
5870 break;
5871 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5872 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5873 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5874 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5875 is_l2 = true;
5876 is_l4 = true;
5877 config.rx_filter = HWTSTAMP_FILTER_SOME;
5878 break;
5879 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5880 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5881 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5882 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5883 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5884 is_l2 = true;
5885 break;
5886 default:
5887 return -ERANGE;
5888 }
5889
5890 if (hw->mac.type == e1000_82575) {
5891 if (tsync_rx_ctl | tsync_tx_ctl)
5892 return -EINVAL;
5893 return 0;
5894 }
5895
5896 /*
5897 * Per-packet timestamping only works if all packets are
5898 * timestamped, so enable timestamping in all packets as
5899 * long as one rx filter was configured.
5900 */
5901 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5902 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5903 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5904 }
5905
5906 /* enable/disable TX */
5907 regval = rd32(E1000_TSYNCTXCTL);
5908 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5909 regval |= tsync_tx_ctl;
5910 wr32(E1000_TSYNCTXCTL, regval);
5911
5912 /* enable/disable RX */
5913 regval = rd32(E1000_TSYNCRXCTL);
5914 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5915 regval |= tsync_rx_ctl;
5916 wr32(E1000_TSYNCRXCTL, regval);
5917
5918 /* define which PTP packets are time stamped */
5919 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5920
5921 /* define ethertype filter for timestamped packets */
5922 if (is_l2)
5923 wr32(E1000_ETQF(3),
5924 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5925 E1000_ETQF_1588 | /* enable timestamping */
5926 ETH_P_1588)); /* 1588 eth protocol type */
5927 else
5928 wr32(E1000_ETQF(3), 0);
5929
5930 #define PTP_PORT 319
5931 /* L4 Queue Filter[3]: filter by destination port and protocol */
5932 if (is_l4) {
5933 u32 ftqf = (IPPROTO_UDP /* UDP */
5934 | E1000_FTQF_VF_BP /* VF not compared */
5935 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5936 | E1000_FTQF_MASK); /* mask all inputs */
5937 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5938
5939 wr32(E1000_IMIR(3), htons(PTP_PORT));
5940 wr32(E1000_IMIREXT(3),
5941 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5942 if (hw->mac.type == e1000_82576) {
5943 /* enable source port check */
5944 wr32(E1000_SPQF(3), htons(PTP_PORT));
5945 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5946 }
5947 wr32(E1000_FTQF(3), ftqf);
5948 } else {
5949 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5950 }
5951 wrfl();
5952
5953 adapter->hwtstamp_config = config;
5954
5955 /* clear TX/RX time stamp registers, just to be sure */
5956 regval = rd32(E1000_TXSTMPH);
5957 regval = rd32(E1000_RXSTMPH);
5958
5959 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5960 -EFAULT : 0;
5961 }
5962
5963 /**
5964 * igb_ioctl -
5965 * @netdev:
5966 * @ifreq:
5967 * @cmd:
5968 **/
5969 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5970 {
5971 switch (cmd) {
5972 case SIOCGMIIPHY:
5973 case SIOCGMIIREG:
5974 case SIOCSMIIREG:
5975 return igb_mii_ioctl(netdev, ifr, cmd);
5976 case SIOCSHWTSTAMP:
5977 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
5978 default:
5979 return -EOPNOTSUPP;
5980 }
5981 }
5982
5983 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5984 {
5985 struct igb_adapter *adapter = hw->back;
5986 u16 cap_offset;
5987
5988 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5989 if (!cap_offset)
5990 return -E1000_ERR_CONFIG;
5991
5992 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5993
5994 return 0;
5995 }
5996
5997 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5998 {
5999 struct igb_adapter *adapter = hw->back;
6000 u16 cap_offset;
6001
6002 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6003 if (!cap_offset)
6004 return -E1000_ERR_CONFIG;
6005
6006 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6007
6008 return 0;
6009 }
6010
6011 static void igb_vlan_rx_register(struct net_device *netdev,
6012 struct vlan_group *grp)
6013 {
6014 struct igb_adapter *adapter = netdev_priv(netdev);
6015 struct e1000_hw *hw = &adapter->hw;
6016 u32 ctrl, rctl;
6017
6018 igb_irq_disable(adapter);
6019 adapter->vlgrp = grp;
6020
6021 if (grp) {
6022 /* enable VLAN tag insert/strip */
6023 ctrl = rd32(E1000_CTRL);
6024 ctrl |= E1000_CTRL_VME;
6025 wr32(E1000_CTRL, ctrl);
6026
6027 /* Disable CFI check */
6028 rctl = rd32(E1000_RCTL);
6029 rctl &= ~E1000_RCTL_CFIEN;
6030 wr32(E1000_RCTL, rctl);
6031 } else {
6032 /* disable VLAN tag insert/strip */
6033 ctrl = rd32(E1000_CTRL);
6034 ctrl &= ~E1000_CTRL_VME;
6035 wr32(E1000_CTRL, ctrl);
6036 }
6037
6038 igb_rlpml_set(adapter);
6039
6040 if (!test_bit(__IGB_DOWN, &adapter->state))
6041 igb_irq_enable(adapter);
6042 }
6043
6044 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6045 {
6046 struct igb_adapter *adapter = netdev_priv(netdev);
6047 struct e1000_hw *hw = &adapter->hw;
6048 int pf_id = adapter->vfs_allocated_count;
6049
6050 /* attempt to add filter to vlvf array */
6051 igb_vlvf_set(adapter, vid, true, pf_id);
6052
6053 /* add the filter since PF can receive vlans w/o entry in vlvf */
6054 igb_vfta_set(hw, vid, true);
6055 }
6056
6057 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6058 {
6059 struct igb_adapter *adapter = netdev_priv(netdev);
6060 struct e1000_hw *hw = &adapter->hw;
6061 int pf_id = adapter->vfs_allocated_count;
6062 s32 err;
6063
6064 igb_irq_disable(adapter);
6065 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6066
6067 if (!test_bit(__IGB_DOWN, &adapter->state))
6068 igb_irq_enable(adapter);
6069
6070 /* remove vlan from VLVF table array */
6071 err = igb_vlvf_set(adapter, vid, false, pf_id);
6072
6073 /* if vid was not present in VLVF just remove it from table */
6074 if (err)
6075 igb_vfta_set(hw, vid, false);
6076 }
6077
6078 static void igb_restore_vlan(struct igb_adapter *adapter)
6079 {
6080 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6081
6082 if (adapter->vlgrp) {
6083 u16 vid;
6084 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
6085 if (!vlan_group_get_device(adapter->vlgrp, vid))
6086 continue;
6087 igb_vlan_rx_add_vid(adapter->netdev, vid);
6088 }
6089 }
6090 }
6091
6092 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6093 {
6094 struct pci_dev *pdev = adapter->pdev;
6095 struct e1000_mac_info *mac = &adapter->hw.mac;
6096
6097 mac->autoneg = 0;
6098
6099 switch (spddplx) {
6100 case SPEED_10 + DUPLEX_HALF:
6101 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6102 break;
6103 case SPEED_10 + DUPLEX_FULL:
6104 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6105 break;
6106 case SPEED_100 + DUPLEX_HALF:
6107 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6108 break;
6109 case SPEED_100 + DUPLEX_FULL:
6110 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6111 break;
6112 case SPEED_1000 + DUPLEX_FULL:
6113 mac->autoneg = 1;
6114 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6115 break;
6116 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6117 default:
6118 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6119 return -EINVAL;
6120 }
6121 return 0;
6122 }
6123
6124 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
6125 {
6126 struct net_device *netdev = pci_get_drvdata(pdev);
6127 struct igb_adapter *adapter = netdev_priv(netdev);
6128 struct e1000_hw *hw = &adapter->hw;
6129 u32 ctrl, rctl, status;
6130 u32 wufc = adapter->wol;
6131 #ifdef CONFIG_PM
6132 int retval = 0;
6133 #endif
6134
6135 netif_device_detach(netdev);
6136
6137 if (netif_running(netdev))
6138 igb_close(netdev);
6139
6140 igb_clear_interrupt_scheme(adapter);
6141
6142 #ifdef CONFIG_PM
6143 retval = pci_save_state(pdev);
6144 if (retval)
6145 return retval;
6146 #endif
6147
6148 status = rd32(E1000_STATUS);
6149 if (status & E1000_STATUS_LU)
6150 wufc &= ~E1000_WUFC_LNKC;
6151
6152 if (wufc) {
6153 igb_setup_rctl(adapter);
6154 igb_set_rx_mode(netdev);
6155
6156 /* turn on all-multi mode if wake on multicast is enabled */
6157 if (wufc & E1000_WUFC_MC) {
6158 rctl = rd32(E1000_RCTL);
6159 rctl |= E1000_RCTL_MPE;
6160 wr32(E1000_RCTL, rctl);
6161 }
6162
6163 ctrl = rd32(E1000_CTRL);
6164 /* advertise wake from D3Cold */
6165 #define E1000_CTRL_ADVD3WUC 0x00100000
6166 /* phy power management enable */
6167 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6168 ctrl |= E1000_CTRL_ADVD3WUC;
6169 wr32(E1000_CTRL, ctrl);
6170
6171 /* Allow time for pending master requests to run */
6172 igb_disable_pcie_master(hw);
6173
6174 wr32(E1000_WUC, E1000_WUC_PME_EN);
6175 wr32(E1000_WUFC, wufc);
6176 } else {
6177 wr32(E1000_WUC, 0);
6178 wr32(E1000_WUFC, 0);
6179 }
6180
6181 *enable_wake = wufc || adapter->en_mng_pt;
6182 if (!*enable_wake)
6183 igb_power_down_link(adapter);
6184 else
6185 igb_power_up_link(adapter);
6186
6187 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6188 * would have already happened in close and is redundant. */
6189 igb_release_hw_control(adapter);
6190
6191 pci_disable_device(pdev);
6192
6193 return 0;
6194 }
6195
6196 #ifdef CONFIG_PM
6197 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6198 {
6199 int retval;
6200 bool wake;
6201
6202 retval = __igb_shutdown(pdev, &wake);
6203 if (retval)
6204 return retval;
6205
6206 if (wake) {
6207 pci_prepare_to_sleep(pdev);
6208 } else {
6209 pci_wake_from_d3(pdev, false);
6210 pci_set_power_state(pdev, PCI_D3hot);
6211 }
6212
6213 return 0;
6214 }
6215
6216 static int igb_resume(struct pci_dev *pdev)
6217 {
6218 struct net_device *netdev = pci_get_drvdata(pdev);
6219 struct igb_adapter *adapter = netdev_priv(netdev);
6220 struct e1000_hw *hw = &adapter->hw;
6221 u32 err;
6222
6223 pci_set_power_state(pdev, PCI_D0);
6224 pci_restore_state(pdev);
6225 pci_save_state(pdev);
6226
6227 err = pci_enable_device_mem(pdev);
6228 if (err) {
6229 dev_err(&pdev->dev,
6230 "igb: Cannot enable PCI device from suspend\n");
6231 return err;
6232 }
6233 pci_set_master(pdev);
6234
6235 pci_enable_wake(pdev, PCI_D3hot, 0);
6236 pci_enable_wake(pdev, PCI_D3cold, 0);
6237
6238 if (igb_init_interrupt_scheme(adapter)) {
6239 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6240 return -ENOMEM;
6241 }
6242
6243 igb_reset(adapter);
6244
6245 /* let the f/w know that the h/w is now under the control of the
6246 * driver. */
6247 igb_get_hw_control(adapter);
6248
6249 wr32(E1000_WUS, ~0);
6250
6251 if (netif_running(netdev)) {
6252 err = igb_open(netdev);
6253 if (err)
6254 return err;
6255 }
6256
6257 netif_device_attach(netdev);
6258
6259 return 0;
6260 }
6261 #endif
6262
6263 static void igb_shutdown(struct pci_dev *pdev)
6264 {
6265 bool wake;
6266
6267 __igb_shutdown(pdev, &wake);
6268
6269 if (system_state == SYSTEM_POWER_OFF) {
6270 pci_wake_from_d3(pdev, wake);
6271 pci_set_power_state(pdev, PCI_D3hot);
6272 }
6273 }
6274
6275 #ifdef CONFIG_NET_POLL_CONTROLLER
6276 /*
6277 * Polling 'interrupt' - used by things like netconsole to send skbs
6278 * without having to re-enable interrupts. It's not called while
6279 * the interrupt routine is executing.
6280 */
6281 static void igb_netpoll(struct net_device *netdev)
6282 {
6283 struct igb_adapter *adapter = netdev_priv(netdev);
6284 struct e1000_hw *hw = &adapter->hw;
6285 int i;
6286
6287 if (!adapter->msix_entries) {
6288 struct igb_q_vector *q_vector = adapter->q_vector[0];
6289 igb_irq_disable(adapter);
6290 napi_schedule(&q_vector->napi);
6291 return;
6292 }
6293
6294 for (i = 0; i < adapter->num_q_vectors; i++) {
6295 struct igb_q_vector *q_vector = adapter->q_vector[i];
6296 wr32(E1000_EIMC, q_vector->eims_value);
6297 napi_schedule(&q_vector->napi);
6298 }
6299 }
6300 #endif /* CONFIG_NET_POLL_CONTROLLER */
6301
6302 /**
6303 * igb_io_error_detected - called when PCI error is detected
6304 * @pdev: Pointer to PCI device
6305 * @state: The current pci connection state
6306 *
6307 * This function is called after a PCI bus error affecting
6308 * this device has been detected.
6309 */
6310 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6311 pci_channel_state_t state)
6312 {
6313 struct net_device *netdev = pci_get_drvdata(pdev);
6314 struct igb_adapter *adapter = netdev_priv(netdev);
6315
6316 netif_device_detach(netdev);
6317
6318 if (state == pci_channel_io_perm_failure)
6319 return PCI_ERS_RESULT_DISCONNECT;
6320
6321 if (netif_running(netdev))
6322 igb_down(adapter);
6323 pci_disable_device(pdev);
6324
6325 /* Request a slot slot reset. */
6326 return PCI_ERS_RESULT_NEED_RESET;
6327 }
6328
6329 /**
6330 * igb_io_slot_reset - called after the pci bus has been reset.
6331 * @pdev: Pointer to PCI device
6332 *
6333 * Restart the card from scratch, as if from a cold-boot. Implementation
6334 * resembles the first-half of the igb_resume routine.
6335 */
6336 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6337 {
6338 struct net_device *netdev = pci_get_drvdata(pdev);
6339 struct igb_adapter *adapter = netdev_priv(netdev);
6340 struct e1000_hw *hw = &adapter->hw;
6341 pci_ers_result_t result;
6342 int err;
6343
6344 if (pci_enable_device_mem(pdev)) {
6345 dev_err(&pdev->dev,
6346 "Cannot re-enable PCI device after reset.\n");
6347 result = PCI_ERS_RESULT_DISCONNECT;
6348 } else {
6349 pci_set_master(pdev);
6350 pci_restore_state(pdev);
6351 pci_save_state(pdev);
6352
6353 pci_enable_wake(pdev, PCI_D3hot, 0);
6354 pci_enable_wake(pdev, PCI_D3cold, 0);
6355
6356 igb_reset(adapter);
6357 wr32(E1000_WUS, ~0);
6358 result = PCI_ERS_RESULT_RECOVERED;
6359 }
6360
6361 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6362 if (err) {
6363 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6364 "failed 0x%0x\n", err);
6365 /* non-fatal, continue */
6366 }
6367
6368 return result;
6369 }
6370
6371 /**
6372 * igb_io_resume - called when traffic can start flowing again.
6373 * @pdev: Pointer to PCI device
6374 *
6375 * This callback is called when the error recovery driver tells us that
6376 * its OK to resume normal operation. Implementation resembles the
6377 * second-half of the igb_resume routine.
6378 */
6379 static void igb_io_resume(struct pci_dev *pdev)
6380 {
6381 struct net_device *netdev = pci_get_drvdata(pdev);
6382 struct igb_adapter *adapter = netdev_priv(netdev);
6383
6384 if (netif_running(netdev)) {
6385 if (igb_up(adapter)) {
6386 dev_err(&pdev->dev, "igb_up failed after reset\n");
6387 return;
6388 }
6389 }
6390
6391 netif_device_attach(netdev);
6392
6393 /* let the f/w know that the h/w is now under the control of the
6394 * driver. */
6395 igb_get_hw_control(adapter);
6396 }
6397
6398 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6399 u8 qsel)
6400 {
6401 u32 rar_low, rar_high;
6402 struct e1000_hw *hw = &adapter->hw;
6403
6404 /* HW expects these in little endian so we reverse the byte order
6405 * from network order (big endian) to little endian
6406 */
6407 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6408 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6409 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6410
6411 /* Indicate to hardware the Address is Valid. */
6412 rar_high |= E1000_RAH_AV;
6413
6414 if (hw->mac.type == e1000_82575)
6415 rar_high |= E1000_RAH_POOL_1 * qsel;
6416 else
6417 rar_high |= E1000_RAH_POOL_1 << qsel;
6418
6419 wr32(E1000_RAL(index), rar_low);
6420 wrfl();
6421 wr32(E1000_RAH(index), rar_high);
6422 wrfl();
6423 }
6424
6425 static int igb_set_vf_mac(struct igb_adapter *adapter,
6426 int vf, unsigned char *mac_addr)
6427 {
6428 struct e1000_hw *hw = &adapter->hw;
6429 /* VF MAC addresses start at end of receive addresses and moves
6430 * torwards the first, as a result a collision should not be possible */
6431 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6432
6433 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6434
6435 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6436
6437 return 0;
6438 }
6439
6440 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6441 {
6442 struct igb_adapter *adapter = netdev_priv(netdev);
6443 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6444 return -EINVAL;
6445 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6446 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6447 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6448 " change effective.");
6449 if (test_bit(__IGB_DOWN, &adapter->state)) {
6450 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6451 " but the PF device is not up.\n");
6452 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6453 " attempting to use the VF device.\n");
6454 }
6455 return igb_set_vf_mac(adapter, vf, mac);
6456 }
6457
6458 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6459 {
6460 return -EOPNOTSUPP;
6461 }
6462
6463 static int igb_ndo_get_vf_config(struct net_device *netdev,
6464 int vf, struct ifla_vf_info *ivi)
6465 {
6466 struct igb_adapter *adapter = netdev_priv(netdev);
6467 if (vf >= adapter->vfs_allocated_count)
6468 return -EINVAL;
6469 ivi->vf = vf;
6470 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6471 ivi->tx_rate = 0;
6472 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6473 ivi->qos = adapter->vf_data[vf].pf_qos;
6474 return 0;
6475 }
6476
6477 static void igb_vmm_control(struct igb_adapter *adapter)
6478 {
6479 struct e1000_hw *hw = &adapter->hw;
6480 u32 reg;
6481
6482 switch (hw->mac.type) {
6483 case e1000_82575:
6484 default:
6485 /* replication is not supported for 82575 */
6486 return;
6487 case e1000_82576:
6488 /* notify HW that the MAC is adding vlan tags */
6489 reg = rd32(E1000_DTXCTL);
6490 reg |= E1000_DTXCTL_VLAN_ADDED;
6491 wr32(E1000_DTXCTL, reg);
6492 case e1000_82580:
6493 /* enable replication vlan tag stripping */
6494 reg = rd32(E1000_RPLOLR);
6495 reg |= E1000_RPLOLR_STRVLAN;
6496 wr32(E1000_RPLOLR, reg);
6497 case e1000_i350:
6498 /* none of the above registers are supported by i350 */
6499 break;
6500 }
6501
6502 if (adapter->vfs_allocated_count) {
6503 igb_vmdq_set_loopback_pf(hw, true);
6504 igb_vmdq_set_replication_pf(hw, true);
6505 } else {
6506 igb_vmdq_set_loopback_pf(hw, false);
6507 igb_vmdq_set_replication_pf(hw, false);
6508 }
6509 }
6510
6511 /* igb_main.c */
Linux kernel - Deliverables
This page took 0.159085 seconds and 6 git commands to generate.