projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
cxgb4: Add support to recognize 40G links
[deliverable/linux.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
1 /*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37 #include <linux/bitmap.h>
38 #include <linux/crc32.h>
39 #include <linux/ctype.h>
40 #include <linux/debugfs.h>
41 #include <linux/err.h>
42 #include <linux/etherdevice.h>
43 #include <linux/firmware.h>
44 #include <linux/if.h>
45 #include <linux/if_vlan.h>
46 #include <linux/init.h>
47 #include <linux/log2.h>
48 #include <linux/mdio.h>
49 #include <linux/module.h>
50 #include <linux/moduleparam.h>
51 #include <linux/mutex.h>
52 #include <linux/netdevice.h>
53 #include <linux/pci.h>
54 #include <linux/aer.h>
55 #include <linux/rtnetlink.h>
56 #include <linux/sched.h>
57 #include <linux/seq_file.h>
58 #include <linux/sockios.h>
59 #include <linux/vmalloc.h>
60 #include <linux/workqueue.h>
61 #include <net/neighbour.h>
62 #include <net/netevent.h>
63 #include <net/addrconf.h>
64 #include <asm/uaccess.h>
65
66 #include "cxgb4.h"
67 #include "t4_regs.h"
68 #include "t4_msg.h"
69 #include "t4fw_api.h"
70 #include "l2t.h"
71
72 #include <../drivers/net/bonding/bonding.h>
73
74 #ifdef DRV_VERSION
75 #undef DRV_VERSION
76 #endif
77 #define DRV_VERSION "2.0.0-ko"
78 #define DRV_DESC "Chelsio T4/T5 Network Driver"
79
80 /*
81 * Max interrupt hold-off timer value in us. Queues fall back to this value
82 * under extreme memory pressure so it's largish to give the system time to
83 * recover.
84 */
85 #define MAX_SGE_TIMERVAL 200U
86
87 enum {
88 /*
89 * Physical Function provisioning constants.
90 */
91 PFRES_NVI = 4, /* # of Virtual Interfaces */
92 PFRES_NETHCTRL = 128, /* # of EQs used for ETH or CTRL Qs */
93 PFRES_NIQFLINT = 128, /* # of ingress Qs/w Free List(s)/intr
94 */
95 PFRES_NEQ = 256, /* # of egress queues */
96 PFRES_NIQ = 0, /* # of ingress queues */
97 PFRES_TC = 0, /* PCI-E traffic class */
98 PFRES_NEXACTF = 128, /* # of exact MPS filters */
99
100 PFRES_R_CAPS = FW_CMD_CAP_PF,
101 PFRES_WX_CAPS = FW_CMD_CAP_PF,
102
103 #ifdef CONFIG_PCI_IOV
104 /*
105 * Virtual Function provisioning constants. We need two extra Ingress
106 * Queues with Interrupt capability to serve as the VF's Firmware
107 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
108 * neither will have Free Lists associated with them). For each
109 * Ethernet/Control Egress Queue and for each Free List, we need an
110 * Egress Context.
111 */
112 VFRES_NPORTS = 1, /* # of "ports" per VF */
113 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
114
115 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
116 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
117 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
118 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
119 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
120 VFRES_TC = 0, /* PCI-E traffic class */
121 VFRES_NEXACTF = 16, /* # of exact MPS filters */
122
123 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
124 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
125 #endif
126 };
127
128 /*
129 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
130 * static and likely not to be useful in the long run. We really need to
131 * implement some form of persistent configuration which the firmware
132 * controls.
133 */
134 static unsigned int pfvfres_pmask(struct adapter *adapter,
135 unsigned int pf, unsigned int vf)
136 {
137 unsigned int portn, portvec;
138
139 /*
140 * Give PF's access to all of the ports.
141 */
142 if (vf == 0)
143 return FW_PFVF_CMD_PMASK_MASK;
144
145 /*
146 * For VFs, we'll assign them access to the ports based purely on the
147 * PF. We assign active ports in order, wrapping around if there are
148 * fewer active ports than PFs: e.g. active port[pf % nports].
149 * Unfortunately the adapter's port_info structs haven't been
150 * initialized yet so we have to compute this.
151 */
152 if (adapter->params.nports == 0)
153 return 0;
154
155 portn = pf % adapter->params.nports;
156 portvec = adapter->params.portvec;
157 for (;;) {
158 /*
159 * Isolate the lowest set bit in the port vector. If we're at
160 * the port number that we want, return that as the pmask.
161 * otherwise mask that bit out of the port vector and
162 * decrement our port number ...
163 */
164 unsigned int pmask = portvec ^ (portvec & (portvec-1));
165 if (portn == 0)
166 return pmask;
167 portn--;
168 portvec &= ~pmask;
169 }
170 /*NOTREACHED*/
171 }
172
173 enum {
174 MAX_TXQ_ENTRIES = 16384,
175 MAX_CTRL_TXQ_ENTRIES = 1024,
176 MAX_RSPQ_ENTRIES = 16384,
177 MAX_RX_BUFFERS = 16384,
178 MIN_TXQ_ENTRIES = 32,
179 MIN_CTRL_TXQ_ENTRIES = 32,
180 MIN_RSPQ_ENTRIES = 128,
181 MIN_FL_ENTRIES = 16
182 };
183
184 /* Host shadow copy of ingress filter entry. This is in host native format
185 * and doesn't match the ordering or bit order, etc. of the hardware of the
186 * firmware command. The use of bit-field structure elements is purely to
187 * remind ourselves of the field size limitations and save memory in the case
188 * where the filter table is large.
189 */
190 struct filter_entry {
191 /* Administrative fields for filter.
192 */
193 u32 valid:1; /* filter allocated and valid */
194 u32 locked:1; /* filter is administratively locked */
195
196 u32 pending:1; /* filter action is pending firmware reply */
197 u32 smtidx:8; /* Source MAC Table index for smac */
198 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
199
200 /* The filter itself. Most of this is a straight copy of information
201 * provided by the extended ioctl(). Some fields are translated to
202 * internal forms -- for instance the Ingress Queue ID passed in from
203 * the ioctl() is translated into the Absolute Ingress Queue ID.
204 */
205 struct ch_filter_specification fs;
206 };
207
208 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
209 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
210 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
211
212 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
213
214 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
215 CH_DEVICE(0xa000, 0), /* PE10K */
216 CH_DEVICE(0x4001, -1),
217 CH_DEVICE(0x4002, -1),
218 CH_DEVICE(0x4003, -1),
219 CH_DEVICE(0x4004, -1),
220 CH_DEVICE(0x4005, -1),
221 CH_DEVICE(0x4006, -1),
222 CH_DEVICE(0x4007, -1),
223 CH_DEVICE(0x4008, -1),
224 CH_DEVICE(0x4009, -1),
225 CH_DEVICE(0x400a, -1),
226 CH_DEVICE(0x4401, 4),
227 CH_DEVICE(0x4402, 4),
228 CH_DEVICE(0x4403, 4),
229 CH_DEVICE(0x4404, 4),
230 CH_DEVICE(0x4405, 4),
231 CH_DEVICE(0x4406, 4),
232 CH_DEVICE(0x4407, 4),
233 CH_DEVICE(0x4408, 4),
234 CH_DEVICE(0x4409, 4),
235 CH_DEVICE(0x440a, 4),
236 CH_DEVICE(0x440d, 4),
237 CH_DEVICE(0x440e, 4),
238 CH_DEVICE(0x5001, 4),
239 CH_DEVICE(0x5002, 4),
240 CH_DEVICE(0x5003, 4),
241 CH_DEVICE(0x5004, 4),
242 CH_DEVICE(0x5005, 4),
243 CH_DEVICE(0x5006, 4),
244 CH_DEVICE(0x5007, 4),
245 CH_DEVICE(0x5008, 4),
246 CH_DEVICE(0x5009, 4),
247 CH_DEVICE(0x500A, 4),
248 CH_DEVICE(0x500B, 4),
249 CH_DEVICE(0x500C, 4),
250 CH_DEVICE(0x500D, 4),
251 CH_DEVICE(0x500E, 4),
252 CH_DEVICE(0x500F, 4),
253 CH_DEVICE(0x5010, 4),
254 CH_DEVICE(0x5011, 4),
255 CH_DEVICE(0x5012, 4),
256 CH_DEVICE(0x5013, 4),
257 CH_DEVICE(0x5401, 4),
258 CH_DEVICE(0x5402, 4),
259 CH_DEVICE(0x5403, 4),
260 CH_DEVICE(0x5404, 4),
261 CH_DEVICE(0x5405, 4),
262 CH_DEVICE(0x5406, 4),
263 CH_DEVICE(0x5407, 4),
264 CH_DEVICE(0x5408, 4),
265 CH_DEVICE(0x5409, 4),
266 CH_DEVICE(0x540A, 4),
267 CH_DEVICE(0x540B, 4),
268 CH_DEVICE(0x540C, 4),
269 CH_DEVICE(0x540D, 4),
270 CH_DEVICE(0x540E, 4),
271 CH_DEVICE(0x540F, 4),
272 CH_DEVICE(0x5410, 4),
273 CH_DEVICE(0x5411, 4),
274 CH_DEVICE(0x5412, 4),
275 CH_DEVICE(0x5413, 4),
276 { 0, }
277 };
278
279 #define FW4_FNAME "cxgb4/t4fw.bin"
280 #define FW5_FNAME "cxgb4/t5fw.bin"
281 #define FW4_CFNAME "cxgb4/t4-config.txt"
282 #define FW5_CFNAME "cxgb4/t5-config.txt"
283
284 MODULE_DESCRIPTION(DRV_DESC);
285 MODULE_AUTHOR("Chelsio Communications");
286 MODULE_LICENSE("Dual BSD/GPL");
287 MODULE_VERSION(DRV_VERSION);
288 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
289 MODULE_FIRMWARE(FW4_FNAME);
290 MODULE_FIRMWARE(FW5_FNAME);
291
292 /*
293 * Normally we're willing to become the firmware's Master PF but will be happy
294 * if another PF has already become the Master and initialized the adapter.
295 * Setting "force_init" will cause this driver to forcibly establish itself as
296 * the Master PF and initialize the adapter.
297 */
298 static uint force_init;
299
300 module_param(force_init, uint, 0644);
301 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
302
303 /*
304 * Normally if the firmware we connect to has Configuration File support, we
305 * use that and only fall back to the old Driver-based initialization if the
306 * Configuration File fails for some reason. If force_old_init is set, then
307 * we'll always use the old Driver-based initialization sequence.
308 */
309 static uint force_old_init;
310
311 module_param(force_old_init, uint, 0644);
312 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
313
314 static int dflt_msg_enable = DFLT_MSG_ENABLE;
315
316 module_param(dflt_msg_enable, int, 0644);
317 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
318
319 /*
320 * The driver uses the best interrupt scheme available on a platform in the
321 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
322 * of these schemes the driver may consider as follows:
323 *
324 * msi = 2: choose from among all three options
325 * msi = 1: only consider MSI and INTx interrupts
326 * msi = 0: force INTx interrupts
327 */
328 static int msi = 2;
329
330 module_param(msi, int, 0644);
331 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
332
333 /*
334 * Queue interrupt hold-off timer values. Queues default to the first of these
335 * upon creation.
336 */
337 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
338
339 module_param_array(intr_holdoff, uint, NULL, 0644);
340 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
341 "0..4 in microseconds");
342
343 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
344
345 module_param_array(intr_cnt, uint, NULL, 0644);
346 MODULE_PARM_DESC(intr_cnt,
347 "thresholds 1..3 for queue interrupt packet counters");
348
349 /*
350 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
351 * offset by 2 bytes in order to have the IP headers line up on 4-byte
352 * boundaries. This is a requirement for many architectures which will throw
353 * a machine check fault if an attempt is made to access one of the 4-byte IP
354 * header fields on a non-4-byte boundary. And it's a major performance issue
355 * even on some architectures which allow it like some implementations of the
356 * x86 ISA. However, some architectures don't mind this and for some very
357 * edge-case performance sensitive applications (like forwarding large volumes
358 * of small packets), setting this DMA offset to 0 will decrease the number of
359 * PCI-E Bus transfers enough to measurably affect performance.
360 */
361 static int rx_dma_offset = 2;
362
363 static bool vf_acls;
364
365 #ifdef CONFIG_PCI_IOV
366 module_param(vf_acls, bool, 0644);
367 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
368
369 /* Configure the number of PCI-E Virtual Function which are to be instantiated
370 * on SR-IOV Capable Physical Functions.
371 */
372 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
373
374 module_param_array(num_vf, uint, NULL, 0644);
375 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
376 #endif
377
378 /*
379 * The filter TCAM has a fixed portion and a variable portion. The fixed
380 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
381 * ports. The variable portion is 36 bits which can include things like Exact
382 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
383 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
384 * far exceed the 36-bit budget for this "compressed" header portion of the
385 * filter. Thus, we have a scarce resource which must be carefully managed.
386 *
387 * By default we set this up to mostly match the set of filter matching
388 * capabilities of T3 but with accommodations for some of T4's more
389 * interesting features:
390 *
391 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
392 * [Inner] VLAN (17), Port (3), FCoE (1) }
393 */
394 enum {
395 TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
396 TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,
397 TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,
398 };
399
400 static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
401
402 module_param(tp_vlan_pri_map, uint, 0644);
403 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");
404
405 static struct dentry *cxgb4_debugfs_root;
406
407 static LIST_HEAD(adapter_list);
408 static DEFINE_MUTEX(uld_mutex);
409 /* Adapter list to be accessed from atomic context */
410 static LIST_HEAD(adap_rcu_list);
411 static DEFINE_SPINLOCK(adap_rcu_lock);
412 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
413 static const char *uld_str[] = { "RDMA", "iSCSI" };
414
415 static void link_report(struct net_device *dev)
416 {
417 if (!netif_carrier_ok(dev))
418 netdev_info(dev, "link down\n");
419 else {
420 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
421
422 const char *s = "10Mbps";
423 const struct port_info *p = netdev_priv(dev);
424
425 switch (p->link_cfg.speed) {
426 case SPEED_10000:
427 s = "10Gbps";
428 break;
429 case SPEED_1000:
430 s = "1000Mbps";
431 break;
432 case SPEED_100:
433 s = "100Mbps";
434 break;
435 case 40000: /* Need a SPEED_40000 in ethtool.h */
436 s = "40Gbps";
437 break;
438 }
439
440 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
441 fc[p->link_cfg.fc]);
442 }
443 }
444
445 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
446 {
447 struct net_device *dev = adapter->port[port_id];
448
449 /* Skip changes from disabled ports. */
450 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
451 if (link_stat)
452 netif_carrier_on(dev);
453 else
454 netif_carrier_off(dev);
455
456 link_report(dev);
457 }
458 }
459
460 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
461 {
462 static const char *mod_str[] = {
463 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
464 };
465
466 const struct net_device *dev = adap->port[port_id];
467 const struct port_info *pi = netdev_priv(dev);
468
469 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
470 netdev_info(dev, "port module unplugged\n");
471 else if (pi->mod_type < ARRAY_SIZE(mod_str))
472 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
473 }
474
475 /*
476 * Configure the exact and hash address filters to handle a port's multicast
477 * and secondary unicast MAC addresses.
478 */
479 static int set_addr_filters(const struct net_device *dev, bool sleep)
480 {
481 u64 mhash = 0;
482 u64 uhash = 0;
483 bool free = true;
484 u16 filt_idx[7];
485 const u8 *addr[7];
486 int ret, naddr = 0;
487 const struct netdev_hw_addr *ha;
488 int uc_cnt = netdev_uc_count(dev);
489 int mc_cnt = netdev_mc_count(dev);
490 const struct port_info *pi = netdev_priv(dev);
491 unsigned int mb = pi->adapter->fn;
492
493 /* first do the secondary unicast addresses */
494 netdev_for_each_uc_addr(ha, dev) {
495 addr[naddr++] = ha->addr;
496 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
497 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
498 naddr, addr, filt_idx, &uhash, sleep);
499 if (ret < 0)
500 return ret;
501
502 free = false;
503 naddr = 0;
504 }
505 }
506
507 /* next set up the multicast addresses */
508 netdev_for_each_mc_addr(ha, dev) {
509 addr[naddr++] = ha->addr;
510 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
511 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
512 naddr, addr, filt_idx, &mhash, sleep);
513 if (ret < 0)
514 return ret;
515
516 free = false;
517 naddr = 0;
518 }
519 }
520
521 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
522 uhash | mhash, sleep);
523 }
524
525 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
526 module_param(dbfifo_int_thresh, int, 0644);
527 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
528
529 /*
530 * usecs to sleep while draining the dbfifo
531 */
532 static int dbfifo_drain_delay = 1000;
533 module_param(dbfifo_drain_delay, int, 0644);
534 MODULE_PARM_DESC(dbfifo_drain_delay,
535 "usecs to sleep while draining the dbfifo");
536
537 /*
538 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
539 * If @mtu is -1 it is left unchanged.
540 */
541 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
542 {
543 int ret;
544 struct port_info *pi = netdev_priv(dev);
545
546 ret = set_addr_filters(dev, sleep_ok);
547 if (ret == 0)
548 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
549 (dev->flags & IFF_PROMISC) ? 1 : 0,
550 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
551 sleep_ok);
552 return ret;
553 }
554
555 static struct workqueue_struct *workq;
556
557 /**
558 * link_start - enable a port
559 * @dev: the port to enable
560 *
561 * Performs the MAC and PHY actions needed to enable a port.
562 */
563 static int link_start(struct net_device *dev)
564 {
565 int ret;
566 struct port_info *pi = netdev_priv(dev);
567 unsigned int mb = pi->adapter->fn;
568
569 /*
570 * We do not set address filters and promiscuity here, the stack does
571 * that step explicitly.
572 */
573 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
574 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
575 if (ret == 0) {
576 ret = t4_change_mac(pi->adapter, mb, pi->viid,
577 pi->xact_addr_filt, dev->dev_addr, true,
578 true);
579 if (ret >= 0) {
580 pi->xact_addr_filt = ret;
581 ret = 0;
582 }
583 }
584 if (ret == 0)
585 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
586 &pi->link_cfg);
587 if (ret == 0)
588 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
589 return ret;
590 }
591
592 /* Clear a filter and release any of its resources that we own. This also
593 * clears the filter's "pending" status.
594 */
595 static void clear_filter(struct adapter *adap, struct filter_entry *f)
596 {
597 /* If the new or old filter have loopback rewriteing rules then we'll
598 * need to free any existing Layer Two Table (L2T) entries of the old
599 * filter rule. The firmware will handle freeing up any Source MAC
600 * Table (SMT) entries used for rewriting Source MAC Addresses in
601 * loopback rules.
602 */
603 if (f->l2t)
604 cxgb4_l2t_release(f->l2t);
605
606 /* The zeroing of the filter rule below clears the filter valid,
607 * pending, locked flags, l2t pointer, etc. so it's all we need for
608 * this operation.
609 */
610 memset(f, 0, sizeof(*f));
611 }
612
613 /* Handle a filter write/deletion reply.
614 */
615 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
616 {
617 unsigned int idx = GET_TID(rpl);
618 unsigned int nidx = idx - adap->tids.ftid_base;
619 unsigned int ret;
620 struct filter_entry *f;
621
622 if (idx >= adap->tids.ftid_base && nidx <
623 (adap->tids.nftids + adap->tids.nsftids)) {
624 idx = nidx;
625 ret = GET_TCB_COOKIE(rpl->cookie);
626 f = &adap->tids.ftid_tab[idx];
627
628 if (ret == FW_FILTER_WR_FLT_DELETED) {
629 /* Clear the filter when we get confirmation from the
630 * hardware that the filter has been deleted.
631 */
632 clear_filter(adap, f);
633 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
634 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
635 idx);
636 clear_filter(adap, f);
637 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
638 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
639 f->pending = 0; /* asynchronous setup completed */
640 f->valid = 1;
641 } else {
642 /* Something went wrong. Issue a warning about the
643 * problem and clear everything out.
644 */
645 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
646 idx, ret);
647 clear_filter(adap, f);
648 }
649 }
650 }
651
652 /* Response queue handler for the FW event queue.
653 */
654 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
655 const struct pkt_gl *gl)
656 {
657 u8 opcode = ((const struct rss_header *)rsp)->opcode;
658
659 rsp++; /* skip RSS header */
660
661 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
662 */
663 if (unlikely(opcode == CPL_FW4_MSG &&
664 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
665 rsp++;
666 opcode = ((const struct rss_header *)rsp)->opcode;
667 rsp++;
668 if (opcode != CPL_SGE_EGR_UPDATE) {
669 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
670 , opcode);
671 goto out;
672 }
673 }
674
675 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
676 const struct cpl_sge_egr_update *p = (void *)rsp;
677 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
678 struct sge_txq *txq;
679
680 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
681 txq->restarts++;
682 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
683 struct sge_eth_txq *eq;
684
685 eq = container_of(txq, struct sge_eth_txq, q);
686 netif_tx_wake_queue(eq->txq);
687 } else {
688 struct sge_ofld_txq *oq;
689
690 oq = container_of(txq, struct sge_ofld_txq, q);
691 tasklet_schedule(&oq->qresume_tsk);
692 }
693 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
694 const struct cpl_fw6_msg *p = (void *)rsp;
695
696 if (p->type == 0)
697 t4_handle_fw_rpl(q->adap, p->data);
698 } else if (opcode == CPL_L2T_WRITE_RPL) {
699 const struct cpl_l2t_write_rpl *p = (void *)rsp;
700
701 do_l2t_write_rpl(q->adap, p);
702 } else if (opcode == CPL_SET_TCB_RPL) {
703 const struct cpl_set_tcb_rpl *p = (void *)rsp;
704
705 filter_rpl(q->adap, p);
706 } else
707 dev_err(q->adap->pdev_dev,
708 "unexpected CPL %#x on FW event queue\n", opcode);
709 out:
710 return 0;
711 }
712
713 /**
714 * uldrx_handler - response queue handler for ULD queues
715 * @q: the response queue that received the packet
716 * @rsp: the response queue descriptor holding the offload message
717 * @gl: the gather list of packet fragments
718 *
719 * Deliver an ingress offload packet to a ULD. All processing is done by
720 * the ULD, we just maintain statistics.
721 */
722 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
723 const struct pkt_gl *gl)
724 {
725 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
726
727 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
728 */
729 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
730 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
731 rsp += 2;
732
733 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
734 rxq->stats.nomem++;
735 return -1;
736 }
737 if (gl == NULL)
738 rxq->stats.imm++;
739 else if (gl == CXGB4_MSG_AN)
740 rxq->stats.an++;
741 else
742 rxq->stats.pkts++;
743 return 0;
744 }
745
746 static void disable_msi(struct adapter *adapter)
747 {
748 if (adapter->flags & USING_MSIX) {
749 pci_disable_msix(adapter->pdev);
750 adapter->flags &= ~USING_MSIX;
751 } else if (adapter->flags & USING_MSI) {
752 pci_disable_msi(adapter->pdev);
753 adapter->flags &= ~USING_MSI;
754 }
755 }
756
757 /*
758 * Interrupt handler for non-data events used with MSI-X.
759 */
760 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
761 {
762 struct adapter *adap = cookie;
763
764 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
765 if (v & PFSW) {
766 adap->swintr = 1;
767 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
768 }
769 t4_slow_intr_handler(adap);
770 return IRQ_HANDLED;
771 }
772
773 /*
774 * Name the MSI-X interrupts.
775 */
776 static void name_msix_vecs(struct adapter *adap)
777 {
778 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
779
780 /* non-data interrupts */
781 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
782
783 /* FW events */
784 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
785 adap->port[0]->name);
786
787 /* Ethernet queues */
788 for_each_port(adap, j) {
789 struct net_device *d = adap->port[j];
790 const struct port_info *pi = netdev_priv(d);
791
792 for (i = 0; i < pi->nqsets; i++, msi_idx++)
793 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
794 d->name, i);
795 }
796
797 /* offload queues */
798 for_each_ofldrxq(&adap->sge, i)
799 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
800 adap->port[0]->name, i);
801
802 for_each_rdmarxq(&adap->sge, i)
803 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
804 adap->port[0]->name, i);
805 }
806
807 static int request_msix_queue_irqs(struct adapter *adap)
808 {
809 struct sge *s = &adap->sge;
810 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi_index = 2;
811
812 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
813 adap->msix_info[1].desc, &s->fw_evtq);
814 if (err)
815 return err;
816
817 for_each_ethrxq(s, ethqidx) {
818 err = request_irq(adap->msix_info[msi_index].vec,
819 t4_sge_intr_msix, 0,
820 adap->msix_info[msi_index].desc,
821 &s->ethrxq[ethqidx].rspq);
822 if (err)
823 goto unwind;
824 msi_index++;
825 }
826 for_each_ofldrxq(s, ofldqidx) {
827 err = request_irq(adap->msix_info[msi_index].vec,
828 t4_sge_intr_msix, 0,
829 adap->msix_info[msi_index].desc,
830 &s->ofldrxq[ofldqidx].rspq);
831 if (err)
832 goto unwind;
833 msi_index++;
834 }
835 for_each_rdmarxq(s, rdmaqidx) {
836 err = request_irq(adap->msix_info[msi_index].vec,
837 t4_sge_intr_msix, 0,
838 adap->msix_info[msi_index].desc,
839 &s->rdmarxq[rdmaqidx].rspq);
840 if (err)
841 goto unwind;
842 msi_index++;
843 }
844 return 0;
845
846 unwind:
847 while (--rdmaqidx >= 0)
848 free_irq(adap->msix_info[--msi_index].vec,
849 &s->rdmarxq[rdmaqidx].rspq);
850 while (--ofldqidx >= 0)
851 free_irq(adap->msix_info[--msi_index].vec,
852 &s->ofldrxq[ofldqidx].rspq);
853 while (--ethqidx >= 0)
854 free_irq(adap->msix_info[--msi_index].vec,
855 &s->ethrxq[ethqidx].rspq);
856 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
857 return err;
858 }
859
860 static void free_msix_queue_irqs(struct adapter *adap)
861 {
862 int i, msi_index = 2;
863 struct sge *s = &adap->sge;
864
865 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
866 for_each_ethrxq(s, i)
867 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
868 for_each_ofldrxq(s, i)
869 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
870 for_each_rdmarxq(s, i)
871 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
872 }
873
874 /**
875 * write_rss - write the RSS table for a given port
876 * @pi: the port
877 * @queues: array of queue indices for RSS
878 *
879 * Sets up the portion of the HW RSS table for the port's VI to distribute
880 * packets to the Rx queues in @queues.
881 */
882 static int write_rss(const struct port_info *pi, const u16 *queues)
883 {
884 u16 *rss;
885 int i, err;
886 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
887
888 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
889 if (!rss)
890 return -ENOMEM;
891
892 /* map the queue indices to queue ids */
893 for (i = 0; i < pi->rss_size; i++, queues++)
894 rss[i] = q[*queues].rspq.abs_id;
895
896 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
897 pi->rss_size, rss, pi->rss_size);
898 kfree(rss);
899 return err;
900 }
901
902 /**
903 * setup_rss - configure RSS
904 * @adap: the adapter
905 *
906 * Sets up RSS for each port.
907 */
908 static int setup_rss(struct adapter *adap)
909 {
910 int i, err;
911
912 for_each_port(adap, i) {
913 const struct port_info *pi = adap2pinfo(adap, i);
914
915 err = write_rss(pi, pi->rss);
916 if (err)
917 return err;
918 }
919 return 0;
920 }
921
922 /*
923 * Return the channel of the ingress queue with the given qid.
924 */
925 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
926 {
927 qid -= p->ingr_start;
928 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
929 }
930
931 /*
932 * Wait until all NAPI handlers are descheduled.
933 */
934 static void quiesce_rx(struct adapter *adap)
935 {
936 int i;
937
938 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
939 struct sge_rspq *q = adap->sge.ingr_map[i];
940
941 if (q && q->handler)
942 napi_disable(&q->napi);
943 }
944 }
945
946 /*
947 * Enable NAPI scheduling and interrupt generation for all Rx queues.
948 */
949 static void enable_rx(struct adapter *adap)
950 {
951 int i;
952
953 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
954 struct sge_rspq *q = adap->sge.ingr_map[i];
955
956 if (!q)
957 continue;
958 if (q->handler)
959 napi_enable(&q->napi);
960 /* 0-increment GTS to start the timer and enable interrupts */
961 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
962 SEINTARM(q->intr_params) |
963 INGRESSQID(q->cntxt_id));
964 }
965 }
966
967 /**
968 * setup_sge_queues - configure SGE Tx/Rx/response queues
969 * @adap: the adapter
970 *
971 * Determines how many sets of SGE queues to use and initializes them.
972 * We support multiple queue sets per port if we have MSI-X, otherwise
973 * just one queue set per port.
974 */
975 static int setup_sge_queues(struct adapter *adap)
976 {
977 int err, msi_idx, i, j;
978 struct sge *s = &adap->sge;
979
980 bitmap_zero(s->starving_fl, MAX_EGRQ);
981 bitmap_zero(s->txq_maperr, MAX_EGRQ);
982
983 if (adap->flags & USING_MSIX)
984 msi_idx = 1; /* vector 0 is for non-queue interrupts */
985 else {
986 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
987 NULL, NULL);
988 if (err)
989 return err;
990 msi_idx = -((int)s->intrq.abs_id + 1);
991 }
992
993 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
994 msi_idx, NULL, fwevtq_handler);
995 if (err) {
996 freeout: t4_free_sge_resources(adap);
997 return err;
998 }
999
1000 for_each_port(adap, i) {
1001 struct net_device *dev = adap->port[i];
1002 struct port_info *pi = netdev_priv(dev);
1003 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1004 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1005
1006 for (j = 0; j < pi->nqsets; j++, q++) {
1007 if (msi_idx > 0)
1008 msi_idx++;
1009 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1010 msi_idx, &q->fl,
1011 t4_ethrx_handler);
1012 if (err)
1013 goto freeout;
1014 q->rspq.idx = j;
1015 memset(&q->stats, 0, sizeof(q->stats));
1016 }
1017 for (j = 0; j < pi->nqsets; j++, t++) {
1018 err = t4_sge_alloc_eth_txq(adap, t, dev,
1019 netdev_get_tx_queue(dev, j),
1020 s->fw_evtq.cntxt_id);
1021 if (err)
1022 goto freeout;
1023 }
1024 }
1025
1026 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1027 for_each_ofldrxq(s, i) {
1028 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1029 struct net_device *dev = adap->port[i / j];
1030
1031 if (msi_idx > 0)
1032 msi_idx++;
1033 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
1034 &q->fl, uldrx_handler);
1035 if (err)
1036 goto freeout;
1037 memset(&q->stats, 0, sizeof(q->stats));
1038 s->ofld_rxq[i] = q->rspq.abs_id;
1039 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1040 s->fw_evtq.cntxt_id);
1041 if (err)
1042 goto freeout;
1043 }
1044
1045 for_each_rdmarxq(s, i) {
1046 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1047
1048 if (msi_idx > 0)
1049 msi_idx++;
1050 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1051 msi_idx, &q->fl, uldrx_handler);
1052 if (err)
1053 goto freeout;
1054 memset(&q->stats, 0, sizeof(q->stats));
1055 s->rdma_rxq[i] = q->rspq.abs_id;
1056 }
1057
1058 for_each_port(adap, i) {
1059 /*
1060 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1061 * have RDMA queues, and that's the right value.
1062 */
1063 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1064 s->fw_evtq.cntxt_id,
1065 s->rdmarxq[i].rspq.cntxt_id);
1066 if (err)
1067 goto freeout;
1068 }
1069
1070 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
1071 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
1072 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
1073 return 0;
1074 }
1075
1076 /*
1077 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1078 * The allocated memory is cleared.
1079 */
1080 void *t4_alloc_mem(size_t size)
1081 {
1082 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
1083
1084 if (!p)
1085 p = vzalloc(size);
1086 return p;
1087 }
1088
1089 /*
1090 * Free memory allocated through alloc_mem().
1091 */
1092 static void t4_free_mem(void *addr)
1093 {
1094 if (is_vmalloc_addr(addr))
1095 vfree(addr);
1096 else
1097 kfree(addr);
1098 }
1099
1100 /* Send a Work Request to write the filter at a specified index. We construct
1101 * a Firmware Filter Work Request to have the work done and put the indicated
1102 * filter into "pending" mode which will prevent any further actions against
1103 * it till we get a reply from the firmware on the completion status of the
1104 * request.
1105 */
1106 static int set_filter_wr(struct adapter *adapter, int fidx)
1107 {
1108 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1109 struct sk_buff *skb;
1110 struct fw_filter_wr *fwr;
1111 unsigned int ftid;
1112
1113 /* If the new filter requires loopback Destination MAC and/or VLAN
1114 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1115 * the filter.
1116 */
1117 if (f->fs.newdmac || f->fs.newvlan) {
1118 /* allocate L2T entry for new filter */
1119 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1120 if (f->l2t == NULL)
1121 return -EAGAIN;
1122 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1123 f->fs.eport, f->fs.dmac)) {
1124 cxgb4_l2t_release(f->l2t);
1125 f->l2t = NULL;
1126 return -ENOMEM;
1127 }
1128 }
1129
1130 ftid = adapter->tids.ftid_base + fidx;
1131
1132 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1133 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1134 memset(fwr, 0, sizeof(*fwr));
1135
1136 /* It would be nice to put most of the following in t4_hw.c but most
1137 * of the work is translating the cxgbtool ch_filter_specification
1138 * into the Work Request and the definition of that structure is
1139 * currently in cxgbtool.h which isn't appropriate to pull into the
1140 * common code. We may eventually try to come up with a more neutral
1141 * filter specification structure but for now it's easiest to simply
1142 * put this fairly direct code in line ...
1143 */
1144 fwr->op_pkd = htonl(FW_WR_OP(FW_FILTER_WR));
1145 fwr->len16_pkd = htonl(FW_WR_LEN16(sizeof(*fwr)/16));
1146 fwr->tid_to_iq =
1147 htonl(V_FW_FILTER_WR_TID(ftid) |
1148 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
1149 V_FW_FILTER_WR_NOREPLY(0) |
1150 V_FW_FILTER_WR_IQ(f->fs.iq));
1151 fwr->del_filter_to_l2tix =
1152 htonl(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
1153 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
1154 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
1155 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
1156 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
1157 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
1158 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
1159 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
1160 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
1161 f->fs.newvlan == VLAN_REWRITE) |
1162 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
1163 f->fs.newvlan == VLAN_REWRITE) |
1164 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
1165 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
1166 V_FW_FILTER_WR_PRIO(f->fs.prio) |
1167 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
1168 fwr->ethtype = htons(f->fs.val.ethtype);
1169 fwr->ethtypem = htons(f->fs.mask.ethtype);
1170 fwr->frag_to_ovlan_vldm =
1171 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
1172 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
1173 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.ivlan_vld) |
1174 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.ovlan_vld) |
1175 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.ivlan_vld) |
1176 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));
1177 fwr->smac_sel = 0;
1178 fwr->rx_chan_rx_rpl_iq =
1179 htons(V_FW_FILTER_WR_RX_CHAN(0) |
1180 V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id));
1181 fwr->maci_to_matchtypem =
1182 htonl(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
1183 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
1184 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
1185 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
1186 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
1187 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
1188 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
1189 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
1190 fwr->ptcl = f->fs.val.proto;
1191 fwr->ptclm = f->fs.mask.proto;
1192 fwr->ttyp = f->fs.val.tos;
1193 fwr->ttypm = f->fs.mask.tos;
1194 fwr->ivlan = htons(f->fs.val.ivlan);
1195 fwr->ivlanm = htons(f->fs.mask.ivlan);
1196 fwr->ovlan = htons(f->fs.val.ovlan);
1197 fwr->ovlanm = htons(f->fs.mask.ovlan);
1198 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1199 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1200 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1201 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1202 fwr->lp = htons(f->fs.val.lport);
1203 fwr->lpm = htons(f->fs.mask.lport);
1204 fwr->fp = htons(f->fs.val.fport);
1205 fwr->fpm = htons(f->fs.mask.fport);
1206 if (f->fs.newsmac)
1207 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1208
1209 /* Mark the filter as "pending" and ship off the Filter Work Request.
1210 * When we get the Work Request Reply we'll clear the pending status.
1211 */
1212 f->pending = 1;
1213 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1214 t4_ofld_send(adapter, skb);
1215 return 0;
1216 }
1217
1218 /* Delete the filter at a specified index.
1219 */
1220 static int del_filter_wr(struct adapter *adapter, int fidx)
1221 {
1222 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1223 struct sk_buff *skb;
1224 struct fw_filter_wr *fwr;
1225 unsigned int len, ftid;
1226
1227 len = sizeof(*fwr);
1228 ftid = adapter->tids.ftid_base + fidx;
1229
1230 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1231 fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1232 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1233
1234 /* Mark the filter as "pending" and ship off the Filter Work Request.
1235 * When we get the Work Request Reply we'll clear the pending status.
1236 */
1237 f->pending = 1;
1238 t4_mgmt_tx(adapter, skb);
1239 return 0;
1240 }
1241
1242 static inline int is_offload(const struct adapter *adap)
1243 {
1244 return adap->params.offload;
1245 }
1246
1247 /*
1248 * Implementation of ethtool operations.
1249 */
1250
1251 static u32 get_msglevel(struct net_device *dev)
1252 {
1253 return netdev2adap(dev)->msg_enable;
1254 }
1255
1256 static void set_msglevel(struct net_device *dev, u32 val)
1257 {
1258 netdev2adap(dev)->msg_enable = val;
1259 }
1260
1261 static char stats_strings[][ETH_GSTRING_LEN] = {
1262 "TxOctetsOK ",
1263 "TxFramesOK ",
1264 "TxBroadcastFrames ",
1265 "TxMulticastFrames ",
1266 "TxUnicastFrames ",
1267 "TxErrorFrames ",
1268
1269 "TxFrames64 ",
1270 "TxFrames65To127 ",
1271 "TxFrames128To255 ",
1272 "TxFrames256To511 ",
1273 "TxFrames512To1023 ",
1274 "TxFrames1024To1518 ",
1275 "TxFrames1519ToMax ",
1276
1277 "TxFramesDropped ",
1278 "TxPauseFrames ",
1279 "TxPPP0Frames ",
1280 "TxPPP1Frames ",
1281 "TxPPP2Frames ",
1282 "TxPPP3Frames ",
1283 "TxPPP4Frames ",
1284 "TxPPP5Frames ",
1285 "TxPPP6Frames ",
1286 "TxPPP7Frames ",
1287
1288 "RxOctetsOK ",
1289 "RxFramesOK ",
1290 "RxBroadcastFrames ",
1291 "RxMulticastFrames ",
1292 "RxUnicastFrames ",
1293
1294 "RxFramesTooLong ",
1295 "RxJabberErrors ",
1296 "RxFCSErrors ",
1297 "RxLengthErrors ",
1298 "RxSymbolErrors ",
1299 "RxRuntFrames ",
1300
1301 "RxFrames64 ",
1302 "RxFrames65To127 ",
1303 "RxFrames128To255 ",
1304 "RxFrames256To511 ",
1305 "RxFrames512To1023 ",
1306 "RxFrames1024To1518 ",
1307 "RxFrames1519ToMax ",
1308
1309 "RxPauseFrames ",
1310 "RxPPP0Frames ",
1311 "RxPPP1Frames ",
1312 "RxPPP2Frames ",
1313 "RxPPP3Frames ",
1314 "RxPPP4Frames ",
1315 "RxPPP5Frames ",
1316 "RxPPP6Frames ",
1317 "RxPPP7Frames ",
1318
1319 "RxBG0FramesDropped ",
1320 "RxBG1FramesDropped ",
1321 "RxBG2FramesDropped ",
1322 "RxBG3FramesDropped ",
1323 "RxBG0FramesTrunc ",
1324 "RxBG1FramesTrunc ",
1325 "RxBG2FramesTrunc ",
1326 "RxBG3FramesTrunc ",
1327
1328 "TSO ",
1329 "TxCsumOffload ",
1330 "RxCsumGood ",
1331 "VLANextractions ",
1332 "VLANinsertions ",
1333 "GROpackets ",
1334 "GROmerged ",
1335 "WriteCoalSuccess ",
1336 "WriteCoalFail ",
1337 };
1338
1339 static int get_sset_count(struct net_device *dev, int sset)
1340 {
1341 switch (sset) {
1342 case ETH_SS_STATS:
1343 return ARRAY_SIZE(stats_strings);
1344 default:
1345 return -EOPNOTSUPP;
1346 }
1347 }
1348
1349 #define T4_REGMAP_SIZE (160 * 1024)
1350 #define T5_REGMAP_SIZE (332 * 1024)
1351
1352 static int get_regs_len(struct net_device *dev)
1353 {
1354 struct adapter *adap = netdev2adap(dev);
1355 if (is_t4(adap->params.chip))
1356 return T4_REGMAP_SIZE;
1357 else
1358 return T5_REGMAP_SIZE;
1359 }
1360
1361 static int get_eeprom_len(struct net_device *dev)
1362 {
1363 return EEPROMSIZE;
1364 }
1365
1366 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1367 {
1368 struct adapter *adapter = netdev2adap(dev);
1369
1370 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1371 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1372 strlcpy(info->bus_info, pci_name(adapter->pdev),
1373 sizeof(info->bus_info));
1374
1375 if (adapter->params.fw_vers)
1376 snprintf(info->fw_version, sizeof(info->fw_version),
1377 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1378 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1379 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1380 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1381 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1382 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1383 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1384 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1385 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1386 }
1387
1388 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1389 {
1390 if (stringset == ETH_SS_STATS)
1391 memcpy(data, stats_strings, sizeof(stats_strings));
1392 }
1393
1394 /*
1395 * port stats maintained per queue of the port. They should be in the same
1396 * order as in stats_strings above.
1397 */
1398 struct queue_port_stats {
1399 u64 tso;
1400 u64 tx_csum;
1401 u64 rx_csum;
1402 u64 vlan_ex;
1403 u64 vlan_ins;
1404 u64 gro_pkts;
1405 u64 gro_merged;
1406 };
1407
1408 static void collect_sge_port_stats(const struct adapter *adap,
1409 const struct port_info *p, struct queue_port_stats *s)
1410 {
1411 int i;
1412 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1413 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1414
1415 memset(s, 0, sizeof(*s));
1416 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1417 s->tso += tx->tso;
1418 s->tx_csum += tx->tx_cso;
1419 s->rx_csum += rx->stats.rx_cso;
1420 s->vlan_ex += rx->stats.vlan_ex;
1421 s->vlan_ins += tx->vlan_ins;
1422 s->gro_pkts += rx->stats.lro_pkts;
1423 s->gro_merged += rx->stats.lro_merged;
1424 }
1425 }
1426
1427 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1428 u64 *data)
1429 {
1430 struct port_info *pi = netdev_priv(dev);
1431 struct adapter *adapter = pi->adapter;
1432 u32 val1, val2;
1433
1434 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1435
1436 data += sizeof(struct port_stats) / sizeof(u64);
1437 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1438 data += sizeof(struct queue_port_stats) / sizeof(u64);
1439 if (!is_t4(adapter->params.chip)) {
1440 t4_write_reg(adapter, SGE_STAT_CFG, STATSOURCE_T5(7));
1441 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL);
1442 val2 = t4_read_reg(adapter, SGE_STAT_MATCH);
1443 *data = val1 - val2;
1444 data++;
1445 *data = val2;
1446 data++;
1447 } else {
1448 memset(data, 0, 2 * sizeof(u64));
1449 *data += 2;
1450 }
1451 }
1452
1453 /*
1454 * Return a version number to identify the type of adapter. The scheme is:
1455 * - bits 0..9: chip version
1456 * - bits 10..15: chip revision
1457 * - bits 16..23: register dump version
1458 */
1459 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1460 {
1461 return CHELSIO_CHIP_VERSION(ap->params.chip) |
1462 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
1463 }
1464
1465 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1466 unsigned int end)
1467 {
1468 u32 *p = buf + start;
1469
1470 for ( ; start <= end; start += sizeof(u32))
1471 *p++ = t4_read_reg(ap, start);
1472 }
1473
1474 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1475 void *buf)
1476 {
1477 static const unsigned int t4_reg_ranges[] = {
1478 0x1008, 0x1108,
1479 0x1180, 0x11b4,
1480 0x11fc, 0x123c,
1481 0x1300, 0x173c,
1482 0x1800, 0x18fc,
1483 0x3000, 0x30d8,
1484 0x30e0, 0x5924,
1485 0x5960, 0x59d4,
1486 0x5a00, 0x5af8,
1487 0x6000, 0x6098,
1488 0x6100, 0x6150,
1489 0x6200, 0x6208,
1490 0x6240, 0x6248,
1491 0x6280, 0x6338,
1492 0x6370, 0x638c,
1493 0x6400, 0x643c,
1494 0x6500, 0x6524,
1495 0x6a00, 0x6a38,
1496 0x6a60, 0x6a78,
1497 0x6b00, 0x6b84,
1498 0x6bf0, 0x6c84,
1499 0x6cf0, 0x6d84,
1500 0x6df0, 0x6e84,
1501 0x6ef0, 0x6f84,
1502 0x6ff0, 0x7084,
1503 0x70f0, 0x7184,
1504 0x71f0, 0x7284,
1505 0x72f0, 0x7384,
1506 0x73f0, 0x7450,
1507 0x7500, 0x7530,
1508 0x7600, 0x761c,
1509 0x7680, 0x76cc,
1510 0x7700, 0x7798,
1511 0x77c0, 0x77fc,
1512 0x7900, 0x79fc,
1513 0x7b00, 0x7c38,
1514 0x7d00, 0x7efc,
1515 0x8dc0, 0x8e1c,
1516 0x8e30, 0x8e78,
1517 0x8ea0, 0x8f6c,
1518 0x8fc0, 0x9074,
1519 0x90fc, 0x90fc,
1520 0x9400, 0x9458,
1521 0x9600, 0x96bc,
1522 0x9800, 0x9808,
1523 0x9820, 0x983c,
1524 0x9850, 0x9864,
1525 0x9c00, 0x9c6c,
1526 0x9c80, 0x9cec,
1527 0x9d00, 0x9d6c,
1528 0x9d80, 0x9dec,
1529 0x9e00, 0x9e6c,
1530 0x9e80, 0x9eec,
1531 0x9f00, 0x9f6c,
1532 0x9f80, 0x9fec,
1533 0xd004, 0xd03c,
1534 0xdfc0, 0xdfe0,
1535 0xe000, 0xea7c,
1536 0xf000, 0x11190,
1537 0x19040, 0x1906c,
1538 0x19078, 0x19080,
1539 0x1908c, 0x19124,
1540 0x19150, 0x191b0,
1541 0x191d0, 0x191e8,
1542 0x19238, 0x1924c,
1543 0x193f8, 0x19474,
1544 0x19490, 0x194f8,
1545 0x19800, 0x19f30,
1546 0x1a000, 0x1a06c,
1547 0x1a0b0, 0x1a120,
1548 0x1a128, 0x1a138,
1549 0x1a190, 0x1a1c4,
1550 0x1a1fc, 0x1a1fc,
1551 0x1e040, 0x1e04c,
1552 0x1e284, 0x1e28c,
1553 0x1e2c0, 0x1e2c0,
1554 0x1e2e0, 0x1e2e0,
1555 0x1e300, 0x1e384,
1556 0x1e3c0, 0x1e3c8,
1557 0x1e440, 0x1e44c,
1558 0x1e684, 0x1e68c,
1559 0x1e6c0, 0x1e6c0,
1560 0x1e6e0, 0x1e6e0,
1561 0x1e700, 0x1e784,
1562 0x1e7c0, 0x1e7c8,
1563 0x1e840, 0x1e84c,
1564 0x1ea84, 0x1ea8c,
1565 0x1eac0, 0x1eac0,
1566 0x1eae0, 0x1eae0,
1567 0x1eb00, 0x1eb84,
1568 0x1ebc0, 0x1ebc8,
1569 0x1ec40, 0x1ec4c,
1570 0x1ee84, 0x1ee8c,
1571 0x1eec0, 0x1eec0,
1572 0x1eee0, 0x1eee0,
1573 0x1ef00, 0x1ef84,
1574 0x1efc0, 0x1efc8,
1575 0x1f040, 0x1f04c,
1576 0x1f284, 0x1f28c,
1577 0x1f2c0, 0x1f2c0,
1578 0x1f2e0, 0x1f2e0,
1579 0x1f300, 0x1f384,
1580 0x1f3c0, 0x1f3c8,
1581 0x1f440, 0x1f44c,
1582 0x1f684, 0x1f68c,
1583 0x1f6c0, 0x1f6c0,
1584 0x1f6e0, 0x1f6e0,
1585 0x1f700, 0x1f784,
1586 0x1f7c0, 0x1f7c8,
1587 0x1f840, 0x1f84c,
1588 0x1fa84, 0x1fa8c,
1589 0x1fac0, 0x1fac0,
1590 0x1fae0, 0x1fae0,
1591 0x1fb00, 0x1fb84,
1592 0x1fbc0, 0x1fbc8,
1593 0x1fc40, 0x1fc4c,
1594 0x1fe84, 0x1fe8c,
1595 0x1fec0, 0x1fec0,
1596 0x1fee0, 0x1fee0,
1597 0x1ff00, 0x1ff84,
1598 0x1ffc0, 0x1ffc8,
1599 0x20000, 0x2002c,
1600 0x20100, 0x2013c,
1601 0x20190, 0x201c8,
1602 0x20200, 0x20318,
1603 0x20400, 0x20528,
1604 0x20540, 0x20614,
1605 0x21000, 0x21040,
1606 0x2104c, 0x21060,
1607 0x210c0, 0x210ec,
1608 0x21200, 0x21268,
1609 0x21270, 0x21284,
1610 0x212fc, 0x21388,
1611 0x21400, 0x21404,
1612 0x21500, 0x21518,
1613 0x2152c, 0x2153c,
1614 0x21550, 0x21554,
1615 0x21600, 0x21600,
1616 0x21608, 0x21628,
1617 0x21630, 0x2163c,
1618 0x21700, 0x2171c,
1619 0x21780, 0x2178c,
1620 0x21800, 0x21c38,
1621 0x21c80, 0x21d7c,
1622 0x21e00, 0x21e04,
1623 0x22000, 0x2202c,
1624 0x22100, 0x2213c,
1625 0x22190, 0x221c8,
1626 0x22200, 0x22318,
1627 0x22400, 0x22528,
1628 0x22540, 0x22614,
1629 0x23000, 0x23040,
1630 0x2304c, 0x23060,
1631 0x230c0, 0x230ec,
1632 0x23200, 0x23268,
1633 0x23270, 0x23284,
1634 0x232fc, 0x23388,
1635 0x23400, 0x23404,
1636 0x23500, 0x23518,
1637 0x2352c, 0x2353c,
1638 0x23550, 0x23554,
1639 0x23600, 0x23600,
1640 0x23608, 0x23628,
1641 0x23630, 0x2363c,
1642 0x23700, 0x2371c,
1643 0x23780, 0x2378c,
1644 0x23800, 0x23c38,
1645 0x23c80, 0x23d7c,
1646 0x23e00, 0x23e04,
1647 0x24000, 0x2402c,
1648 0x24100, 0x2413c,
1649 0x24190, 0x241c8,
1650 0x24200, 0x24318,
1651 0x24400, 0x24528,
1652 0x24540, 0x24614,
1653 0x25000, 0x25040,
1654 0x2504c, 0x25060,
1655 0x250c0, 0x250ec,
1656 0x25200, 0x25268,
1657 0x25270, 0x25284,
1658 0x252fc, 0x25388,
1659 0x25400, 0x25404,
1660 0x25500, 0x25518,
1661 0x2552c, 0x2553c,
1662 0x25550, 0x25554,
1663 0x25600, 0x25600,
1664 0x25608, 0x25628,
1665 0x25630, 0x2563c,
1666 0x25700, 0x2571c,
1667 0x25780, 0x2578c,
1668 0x25800, 0x25c38,
1669 0x25c80, 0x25d7c,
1670 0x25e00, 0x25e04,
1671 0x26000, 0x2602c,
1672 0x26100, 0x2613c,
1673 0x26190, 0x261c8,
1674 0x26200, 0x26318,
1675 0x26400, 0x26528,
1676 0x26540, 0x26614,
1677 0x27000, 0x27040,
1678 0x2704c, 0x27060,
1679 0x270c0, 0x270ec,
1680 0x27200, 0x27268,
1681 0x27270, 0x27284,
1682 0x272fc, 0x27388,
1683 0x27400, 0x27404,
1684 0x27500, 0x27518,
1685 0x2752c, 0x2753c,
1686 0x27550, 0x27554,
1687 0x27600, 0x27600,
1688 0x27608, 0x27628,
1689 0x27630, 0x2763c,
1690 0x27700, 0x2771c,
1691 0x27780, 0x2778c,
1692 0x27800, 0x27c38,
1693 0x27c80, 0x27d7c,
1694 0x27e00, 0x27e04
1695 };
1696
1697 static const unsigned int t5_reg_ranges[] = {
1698 0x1008, 0x1148,
1699 0x1180, 0x11b4,
1700 0x11fc, 0x123c,
1701 0x1280, 0x173c,
1702 0x1800, 0x18fc,
1703 0x3000, 0x3028,
1704 0x3060, 0x30d8,
1705 0x30e0, 0x30fc,
1706 0x3140, 0x357c,
1707 0x35a8, 0x35cc,
1708 0x35ec, 0x35ec,
1709 0x3600, 0x5624,
1710 0x56cc, 0x575c,
1711 0x580c, 0x5814,
1712 0x5890, 0x58bc,
1713 0x5940, 0x59dc,
1714 0x59fc, 0x5a18,
1715 0x5a60, 0x5a9c,
1716 0x5b9c, 0x5bfc,
1717 0x6000, 0x6040,
1718 0x6058, 0x614c,
1719 0x7700, 0x7798,
1720 0x77c0, 0x78fc,
1721 0x7b00, 0x7c54,
1722 0x7d00, 0x7efc,
1723 0x8dc0, 0x8de0,
1724 0x8df8, 0x8e84,
1725 0x8ea0, 0x8f84,
1726 0x8fc0, 0x90f8,
1727 0x9400, 0x9470,
1728 0x9600, 0x96f4,
1729 0x9800, 0x9808,
1730 0x9820, 0x983c,
1731 0x9850, 0x9864,
1732 0x9c00, 0x9c6c,
1733 0x9c80, 0x9cec,
1734 0x9d00, 0x9d6c,
1735 0x9d80, 0x9dec,
1736 0x9e00, 0x9e6c,
1737 0x9e80, 0x9eec,
1738 0x9f00, 0x9f6c,
1739 0x9f80, 0xa020,
1740 0xd004, 0xd03c,
1741 0xdfc0, 0xdfe0,
1742 0xe000, 0x11088,
1743 0x1109c, 0x1117c,
1744 0x11190, 0x11204,
1745 0x19040, 0x1906c,
1746 0x19078, 0x19080,
1747 0x1908c, 0x19124,
1748 0x19150, 0x191b0,
1749 0x191d0, 0x191e8,
1750 0x19238, 0x19290,
1751 0x193f8, 0x19474,
1752 0x19490, 0x194cc,
1753 0x194f0, 0x194f8,
1754 0x19c00, 0x19c60,
1755 0x19c94, 0x19e10,
1756 0x19e50, 0x19f34,
1757 0x19f40, 0x19f50,
1758 0x19f90, 0x19fe4,
1759 0x1a000, 0x1a06c,
1760 0x1a0b0, 0x1a120,
1761 0x1a128, 0x1a138,
1762 0x1a190, 0x1a1c4,
1763 0x1a1fc, 0x1a1fc,
1764 0x1e008, 0x1e00c,
1765 0x1e040, 0x1e04c,
1766 0x1e284, 0x1e290,
1767 0x1e2c0, 0x1e2c0,
1768 0x1e2e0, 0x1e2e0,
1769 0x1e300, 0x1e384,
1770 0x1e3c0, 0x1e3c8,
1771 0x1e408, 0x1e40c,
1772 0x1e440, 0x1e44c,
1773 0x1e684, 0x1e690,
1774 0x1e6c0, 0x1e6c0,
1775 0x1e6e0, 0x1e6e0,
1776 0x1e700, 0x1e784,
1777 0x1e7c0, 0x1e7c8,
1778 0x1e808, 0x1e80c,
1779 0x1e840, 0x1e84c,
1780 0x1ea84, 0x1ea90,
1781 0x1eac0, 0x1eac0,
1782 0x1eae0, 0x1eae0,
1783 0x1eb00, 0x1eb84,
1784 0x1ebc0, 0x1ebc8,
1785 0x1ec08, 0x1ec0c,
1786 0x1ec40, 0x1ec4c,
1787 0x1ee84, 0x1ee90,
1788 0x1eec0, 0x1eec0,
1789 0x1eee0, 0x1eee0,
1790 0x1ef00, 0x1ef84,
1791 0x1efc0, 0x1efc8,
1792 0x1f008, 0x1f00c,
1793 0x1f040, 0x1f04c,
1794 0x1f284, 0x1f290,
1795 0x1f2c0, 0x1f2c0,
1796 0x1f2e0, 0x1f2e0,
1797 0x1f300, 0x1f384,
1798 0x1f3c0, 0x1f3c8,
1799 0x1f408, 0x1f40c,
1800 0x1f440, 0x1f44c,
1801 0x1f684, 0x1f690,
1802 0x1f6c0, 0x1f6c0,
1803 0x1f6e0, 0x1f6e0,
1804 0x1f700, 0x1f784,
1805 0x1f7c0, 0x1f7c8,
1806 0x1f808, 0x1f80c,
1807 0x1f840, 0x1f84c,
1808 0x1fa84, 0x1fa90,
1809 0x1fac0, 0x1fac0,
1810 0x1fae0, 0x1fae0,
1811 0x1fb00, 0x1fb84,
1812 0x1fbc0, 0x1fbc8,
1813 0x1fc08, 0x1fc0c,
1814 0x1fc40, 0x1fc4c,
1815 0x1fe84, 0x1fe90,
1816 0x1fec0, 0x1fec0,
1817 0x1fee0, 0x1fee0,
1818 0x1ff00, 0x1ff84,
1819 0x1ffc0, 0x1ffc8,
1820 0x30000, 0x30030,
1821 0x30100, 0x30144,
1822 0x30190, 0x301d0,
1823 0x30200, 0x30318,
1824 0x30400, 0x3052c,
1825 0x30540, 0x3061c,
1826 0x30800, 0x30834,
1827 0x308c0, 0x30908,
1828 0x30910, 0x309ac,
1829 0x30a00, 0x30a04,
1830 0x30a0c, 0x30a2c,
1831 0x30a44, 0x30a50,
1832 0x30a74, 0x30c24,
1833 0x30d08, 0x30d14,
1834 0x30d1c, 0x30d20,
1835 0x30d3c, 0x30d50,
1836 0x31200, 0x3120c,
1837 0x31220, 0x31220,
1838 0x31240, 0x31240,
1839 0x31600, 0x31600,
1840 0x31608, 0x3160c,
1841 0x31a00, 0x31a1c,
1842 0x31e04, 0x31e20,
1843 0x31e38, 0x31e3c,
1844 0x31e80, 0x31e80,
1845 0x31e88, 0x31ea8,
1846 0x31eb0, 0x31eb4,
1847 0x31ec8, 0x31ed4,
1848 0x31fb8, 0x32004,
1849 0x32208, 0x3223c,
1850 0x32600, 0x32630,
1851 0x32a00, 0x32abc,
1852 0x32b00, 0x32b70,
1853 0x33000, 0x33048,
1854 0x33060, 0x3309c,
1855 0x330f0, 0x33148,
1856 0x33160, 0x3319c,
1857 0x331f0, 0x332e4,
1858 0x332f8, 0x333e4,
1859 0x333f8, 0x33448,
1860 0x33460, 0x3349c,
1861 0x334f0, 0x33548,
1862 0x33560, 0x3359c,
1863 0x335f0, 0x336e4,
1864 0x336f8, 0x337e4,
1865 0x337f8, 0x337fc,
1866 0x33814, 0x33814,
1867 0x3382c, 0x3382c,
1868 0x33880, 0x3388c,
1869 0x338e8, 0x338ec,
1870 0x33900, 0x33948,
1871 0x33960, 0x3399c,
1872 0x339f0, 0x33ae4,
1873 0x33af8, 0x33b10,
1874 0x33b28, 0x33b28,
1875 0x33b3c, 0x33b50,
1876 0x33bf0, 0x33c10,
1877 0x33c28, 0x33c28,
1878 0x33c3c, 0x33c50,
1879 0x33cf0, 0x33cfc,
1880 0x34000, 0x34030,
1881 0x34100, 0x34144,
1882 0x34190, 0x341d0,
1883 0x34200, 0x34318,
1884 0x34400, 0x3452c,
1885 0x34540, 0x3461c,
1886 0x34800, 0x34834,
1887 0x348c0, 0x34908,
1888 0x34910, 0x349ac,
1889 0x34a00, 0x34a04,
1890 0x34a0c, 0x34a2c,
1891 0x34a44, 0x34a50,
1892 0x34a74, 0x34c24,
1893 0x34d08, 0x34d14,
1894 0x34d1c, 0x34d20,
1895 0x34d3c, 0x34d50,
1896 0x35200, 0x3520c,
1897 0x35220, 0x35220,
1898 0x35240, 0x35240,
1899 0x35600, 0x35600,
1900 0x35608, 0x3560c,
1901 0x35a00, 0x35a1c,
1902 0x35e04, 0x35e20,
1903 0x35e38, 0x35e3c,
1904 0x35e80, 0x35e80,
1905 0x35e88, 0x35ea8,
1906 0x35eb0, 0x35eb4,
1907 0x35ec8, 0x35ed4,
1908 0x35fb8, 0x36004,
1909 0x36208, 0x3623c,
1910 0x36600, 0x36630,
1911 0x36a00, 0x36abc,
1912 0x36b00, 0x36b70,
1913 0x37000, 0x37048,
1914 0x37060, 0x3709c,
1915 0x370f0, 0x37148,
1916 0x37160, 0x3719c,
1917 0x371f0, 0x372e4,
1918 0x372f8, 0x373e4,
1919 0x373f8, 0x37448,
1920 0x37460, 0x3749c,
1921 0x374f0, 0x37548,
1922 0x37560, 0x3759c,
1923 0x375f0, 0x376e4,
1924 0x376f8, 0x377e4,
1925 0x377f8, 0x377fc,
1926 0x37814, 0x37814,
1927 0x3782c, 0x3782c,
1928 0x37880, 0x3788c,
1929 0x378e8, 0x378ec,
1930 0x37900, 0x37948,
1931 0x37960, 0x3799c,
1932 0x379f0, 0x37ae4,
1933 0x37af8, 0x37b10,
1934 0x37b28, 0x37b28,
1935 0x37b3c, 0x37b50,
1936 0x37bf0, 0x37c10,
1937 0x37c28, 0x37c28,
1938 0x37c3c, 0x37c50,
1939 0x37cf0, 0x37cfc,
1940 0x38000, 0x38030,
1941 0x38100, 0x38144,
1942 0x38190, 0x381d0,
1943 0x38200, 0x38318,
1944 0x38400, 0x3852c,
1945 0x38540, 0x3861c,
1946 0x38800, 0x38834,
1947 0x388c0, 0x38908,
1948 0x38910, 0x389ac,
1949 0x38a00, 0x38a04,
1950 0x38a0c, 0x38a2c,
1951 0x38a44, 0x38a50,
1952 0x38a74, 0x38c24,
1953 0x38d08, 0x38d14,
1954 0x38d1c, 0x38d20,
1955 0x38d3c, 0x38d50,
1956 0x39200, 0x3920c,
1957 0x39220, 0x39220,
1958 0x39240, 0x39240,
1959 0x39600, 0x39600,
1960 0x39608, 0x3960c,
1961 0x39a00, 0x39a1c,
1962 0x39e04, 0x39e20,
1963 0x39e38, 0x39e3c,
1964 0x39e80, 0x39e80,
1965 0x39e88, 0x39ea8,
1966 0x39eb0, 0x39eb4,
1967 0x39ec8, 0x39ed4,
1968 0x39fb8, 0x3a004,
1969 0x3a208, 0x3a23c,
1970 0x3a600, 0x3a630,
1971 0x3aa00, 0x3aabc,
1972 0x3ab00, 0x3ab70,
1973 0x3b000, 0x3b048,
1974 0x3b060, 0x3b09c,
1975 0x3b0f0, 0x3b148,
1976 0x3b160, 0x3b19c,
1977 0x3b1f0, 0x3b2e4,
1978 0x3b2f8, 0x3b3e4,
1979 0x3b3f8, 0x3b448,
1980 0x3b460, 0x3b49c,
1981 0x3b4f0, 0x3b548,
1982 0x3b560, 0x3b59c,
1983 0x3b5f0, 0x3b6e4,
1984 0x3b6f8, 0x3b7e4,
1985 0x3b7f8, 0x3b7fc,
1986 0x3b814, 0x3b814,
1987 0x3b82c, 0x3b82c,
1988 0x3b880, 0x3b88c,
1989 0x3b8e8, 0x3b8ec,
1990 0x3b900, 0x3b948,
1991 0x3b960, 0x3b99c,
1992 0x3b9f0, 0x3bae4,
1993 0x3baf8, 0x3bb10,
1994 0x3bb28, 0x3bb28,
1995 0x3bb3c, 0x3bb50,
1996 0x3bbf0, 0x3bc10,
1997 0x3bc28, 0x3bc28,
1998 0x3bc3c, 0x3bc50,
1999 0x3bcf0, 0x3bcfc,
2000 0x3c000, 0x3c030,
2001 0x3c100, 0x3c144,
2002 0x3c190, 0x3c1d0,
2003 0x3c200, 0x3c318,
2004 0x3c400, 0x3c52c,
2005 0x3c540, 0x3c61c,
2006 0x3c800, 0x3c834,
2007 0x3c8c0, 0x3c908,
2008 0x3c910, 0x3c9ac,
2009 0x3ca00, 0x3ca04,
2010 0x3ca0c, 0x3ca2c,
2011 0x3ca44, 0x3ca50,
2012 0x3ca74, 0x3cc24,
2013 0x3cd08, 0x3cd14,
2014 0x3cd1c, 0x3cd20,
2015 0x3cd3c, 0x3cd50,
2016 0x3d200, 0x3d20c,
2017 0x3d220, 0x3d220,
2018 0x3d240, 0x3d240,
2019 0x3d600, 0x3d600,
2020 0x3d608, 0x3d60c,
2021 0x3da00, 0x3da1c,
2022 0x3de04, 0x3de20,
2023 0x3de38, 0x3de3c,
2024 0x3de80, 0x3de80,
2025 0x3de88, 0x3dea8,
2026 0x3deb0, 0x3deb4,
2027 0x3dec8, 0x3ded4,
2028 0x3dfb8, 0x3e004,
2029 0x3e208, 0x3e23c,
2030 0x3e600, 0x3e630,
2031 0x3ea00, 0x3eabc,
2032 0x3eb00, 0x3eb70,
2033 0x3f000, 0x3f048,
2034 0x3f060, 0x3f09c,
2035 0x3f0f0, 0x3f148,
2036 0x3f160, 0x3f19c,
2037 0x3f1f0, 0x3f2e4,
2038 0x3f2f8, 0x3f3e4,
2039 0x3f3f8, 0x3f448,
2040 0x3f460, 0x3f49c,
2041 0x3f4f0, 0x3f548,
2042 0x3f560, 0x3f59c,
2043 0x3f5f0, 0x3f6e4,
2044 0x3f6f8, 0x3f7e4,
2045 0x3f7f8, 0x3f7fc,
2046 0x3f814, 0x3f814,
2047 0x3f82c, 0x3f82c,
2048 0x3f880, 0x3f88c,
2049 0x3f8e8, 0x3f8ec,
2050 0x3f900, 0x3f948,
2051 0x3f960, 0x3f99c,
2052 0x3f9f0, 0x3fae4,
2053 0x3faf8, 0x3fb10,
2054 0x3fb28, 0x3fb28,
2055 0x3fb3c, 0x3fb50,
2056 0x3fbf0, 0x3fc10,
2057 0x3fc28, 0x3fc28,
2058 0x3fc3c, 0x3fc50,
2059 0x3fcf0, 0x3fcfc,
2060 0x40000, 0x4000c,
2061 0x40040, 0x40068,
2062 0x40080, 0x40144,
2063 0x40180, 0x4018c,
2064 0x40200, 0x40298,
2065 0x402ac, 0x4033c,
2066 0x403f8, 0x403fc,
2067 0x41300, 0x413c4,
2068 0x41400, 0x4141c,
2069 0x41480, 0x414d0,
2070 0x44000, 0x44078,
2071 0x440c0, 0x44278,
2072 0x442c0, 0x44478,
2073 0x444c0, 0x44678,
2074 0x446c0, 0x44878,
2075 0x448c0, 0x449fc,
2076 0x45000, 0x45068,
2077 0x45080, 0x45084,
2078 0x450a0, 0x450b0,
2079 0x45200, 0x45268,
2080 0x45280, 0x45284,
2081 0x452a0, 0x452b0,
2082 0x460c0, 0x460e4,
2083 0x47000, 0x4708c,
2084 0x47200, 0x47250,
2085 0x47400, 0x47420,
2086 0x47600, 0x47618,
2087 0x47800, 0x47814,
2088 0x48000, 0x4800c,
2089 0x48040, 0x48068,
2090 0x48080, 0x48144,
2091 0x48180, 0x4818c,
2092 0x48200, 0x48298,
2093 0x482ac, 0x4833c,
2094 0x483f8, 0x483fc,
2095 0x49300, 0x493c4,
2096 0x49400, 0x4941c,
2097 0x49480, 0x494d0,
2098 0x4c000, 0x4c078,
2099 0x4c0c0, 0x4c278,
2100 0x4c2c0, 0x4c478,
2101 0x4c4c0, 0x4c678,
2102 0x4c6c0, 0x4c878,
2103 0x4c8c0, 0x4c9fc,
2104 0x4d000, 0x4d068,
2105 0x4d080, 0x4d084,
2106 0x4d0a0, 0x4d0b0,
2107 0x4d200, 0x4d268,
2108 0x4d280, 0x4d284,
2109 0x4d2a0, 0x4d2b0,
2110 0x4e0c0, 0x4e0e4,
2111 0x4f000, 0x4f08c,
2112 0x4f200, 0x4f250,
2113 0x4f400, 0x4f420,
2114 0x4f600, 0x4f618,
2115 0x4f800, 0x4f814,
2116 0x50000, 0x500cc,
2117 0x50400, 0x50400,
2118 0x50800, 0x508cc,
2119 0x50c00, 0x50c00,
2120 0x51000, 0x5101c,
2121 0x51300, 0x51308,
2122 };
2123
2124 int i;
2125 struct adapter *ap = netdev2adap(dev);
2126 static const unsigned int *reg_ranges;
2127 int arr_size = 0, buf_size = 0;
2128
2129 if (is_t4(ap->params.chip)) {
2130 reg_ranges = &t4_reg_ranges[0];
2131 arr_size = ARRAY_SIZE(t4_reg_ranges);
2132 buf_size = T4_REGMAP_SIZE;
2133 } else {
2134 reg_ranges = &t5_reg_ranges[0];
2135 arr_size = ARRAY_SIZE(t5_reg_ranges);
2136 buf_size = T5_REGMAP_SIZE;
2137 }
2138
2139 regs->version = mk_adap_vers(ap);
2140
2141 memset(buf, 0, buf_size);
2142 for (i = 0; i < arr_size; i += 2)
2143 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2144 }
2145
2146 static int restart_autoneg(struct net_device *dev)
2147 {
2148 struct port_info *p = netdev_priv(dev);
2149
2150 if (!netif_running(dev))
2151 return -EAGAIN;
2152 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2153 return -EINVAL;
2154 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
2155 return 0;
2156 }
2157
2158 static int identify_port(struct net_device *dev,
2159 enum ethtool_phys_id_state state)
2160 {
2161 unsigned int val;
2162 struct adapter *adap = netdev2adap(dev);
2163
2164 if (state == ETHTOOL_ID_ACTIVE)
2165 val = 0xffff;
2166 else if (state == ETHTOOL_ID_INACTIVE)
2167 val = 0;
2168 else
2169 return -EINVAL;
2170
2171 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
2172 }
2173
2174 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
2175 {
2176 unsigned int v = 0;
2177
2178 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2179 type == FW_PORT_TYPE_BT_XAUI) {
2180 v |= SUPPORTED_TP;
2181 if (caps & FW_PORT_CAP_SPEED_100M)
2182 v |= SUPPORTED_100baseT_Full;
2183 if (caps & FW_PORT_CAP_SPEED_1G)
2184 v |= SUPPORTED_1000baseT_Full;
2185 if (caps & FW_PORT_CAP_SPEED_10G)
2186 v |= SUPPORTED_10000baseT_Full;
2187 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2188 v |= SUPPORTED_Backplane;
2189 if (caps & FW_PORT_CAP_SPEED_1G)
2190 v |= SUPPORTED_1000baseKX_Full;
2191 if (caps & FW_PORT_CAP_SPEED_10G)
2192 v |= SUPPORTED_10000baseKX4_Full;
2193 } else if (type == FW_PORT_TYPE_KR)
2194 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
2195 else if (type == FW_PORT_TYPE_BP_AP)
2196 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2197 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2198 else if (type == FW_PORT_TYPE_BP4_AP)
2199 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2200 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2201 SUPPORTED_10000baseKX4_Full;
2202 else if (type == FW_PORT_TYPE_FIBER_XFI ||
2203 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
2204 v |= SUPPORTED_FIBRE;
2205 else if (type == FW_PORT_TYPE_BP40_BA)
2206 v |= SUPPORTED_40000baseSR4_Full;
2207
2208 if (caps & FW_PORT_CAP_ANEG)
2209 v |= SUPPORTED_Autoneg;
2210 return v;
2211 }
2212
2213 static unsigned int to_fw_linkcaps(unsigned int caps)
2214 {
2215 unsigned int v = 0;
2216
2217 if (caps & ADVERTISED_100baseT_Full)
2218 v |= FW_PORT_CAP_SPEED_100M;
2219 if (caps & ADVERTISED_1000baseT_Full)
2220 v |= FW_PORT_CAP_SPEED_1G;
2221 if (caps & ADVERTISED_10000baseT_Full)
2222 v |= FW_PORT_CAP_SPEED_10G;
2223 if (caps & ADVERTISED_40000baseSR4_Full)
2224 v |= FW_PORT_CAP_SPEED_40G;
2225 return v;
2226 }
2227
2228 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2229 {
2230 const struct port_info *p = netdev_priv(dev);
2231
2232 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
2233 p->port_type == FW_PORT_TYPE_BT_XFI ||
2234 p->port_type == FW_PORT_TYPE_BT_XAUI)
2235 cmd->port = PORT_TP;
2236 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2237 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
2238 cmd->port = PORT_FIBRE;
2239 else if (p->port_type == FW_PORT_TYPE_SFP) {
2240 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2241 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
2242 cmd->port = PORT_DA;
2243 else
2244 cmd->port = PORT_FIBRE;
2245 } else
2246 cmd->port = PORT_OTHER;
2247
2248 if (p->mdio_addr >= 0) {
2249 cmd->phy_address = p->mdio_addr;
2250 cmd->transceiver = XCVR_EXTERNAL;
2251 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2252 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2253 } else {
2254 cmd->phy_address = 0; /* not really, but no better option */
2255 cmd->transceiver = XCVR_INTERNAL;
2256 cmd->mdio_support = 0;
2257 }
2258
2259 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2260 cmd->advertising = from_fw_linkcaps(p->port_type,
2261 p->link_cfg.advertising);
2262 ethtool_cmd_speed_set(cmd,
2263 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
2264 cmd->duplex = DUPLEX_FULL;
2265 cmd->autoneg = p->link_cfg.autoneg;
2266 cmd->maxtxpkt = 0;
2267 cmd->maxrxpkt = 0;
2268 return 0;
2269 }
2270
2271 static unsigned int speed_to_caps(int speed)
2272 {
2273 if (speed == SPEED_100)
2274 return FW_PORT_CAP_SPEED_100M;
2275 if (speed == SPEED_1000)
2276 return FW_PORT_CAP_SPEED_1G;
2277 if (speed == SPEED_10000)
2278 return FW_PORT_CAP_SPEED_10G;
2279 if (speed == 40000) /* Need SPEED_40000 in ethtool.h */
2280 return FW_PORT_CAP_SPEED_40G;
2281 return 0;
2282 }
2283
2284 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2285 {
2286 unsigned int cap;
2287 struct port_info *p = netdev_priv(dev);
2288 struct link_config *lc = &p->link_cfg;
2289 u32 speed = ethtool_cmd_speed(cmd);
2290
2291 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
2292 return -EINVAL;
2293
2294 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2295 /*
2296 * PHY offers a single speed. See if that's what's
2297 * being requested.
2298 */
2299 if (cmd->autoneg == AUTONEG_DISABLE &&
2300 (lc->supported & speed_to_caps(speed)))
2301 return 0;
2302 return -EINVAL;
2303 }
2304
2305 if (cmd->autoneg == AUTONEG_DISABLE) {
2306 cap = speed_to_caps(speed);
2307
2308 if (!(lc->supported & cap) ||
2309 (speed == SPEED_1000) ||
2310 (speed == SPEED_10000) ||
2311 (speed == 40000))
2312 return -EINVAL;
2313 lc->requested_speed = cap;
2314 lc->advertising = 0;
2315 } else {
2316 cap = to_fw_linkcaps(cmd->advertising);
2317 if (!(lc->supported & cap))
2318 return -EINVAL;
2319 lc->requested_speed = 0;
2320 lc->advertising = cap | FW_PORT_CAP_ANEG;
2321 }
2322 lc->autoneg = cmd->autoneg;
2323
2324 if (netif_running(dev))
2325 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2326 lc);
2327 return 0;
2328 }
2329
2330 static void get_pauseparam(struct net_device *dev,
2331 struct ethtool_pauseparam *epause)
2332 {
2333 struct port_info *p = netdev_priv(dev);
2334
2335 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2336 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2337 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2338 }
2339
2340 static int set_pauseparam(struct net_device *dev,
2341 struct ethtool_pauseparam *epause)
2342 {
2343 struct port_info *p = netdev_priv(dev);
2344 struct link_config *lc = &p->link_cfg;
2345
2346 if (epause->autoneg == AUTONEG_DISABLE)
2347 lc->requested_fc = 0;
2348 else if (lc->supported & FW_PORT_CAP_ANEG)
2349 lc->requested_fc = PAUSE_AUTONEG;
2350 else
2351 return -EINVAL;
2352
2353 if (epause->rx_pause)
2354 lc->requested_fc |= PAUSE_RX;
2355 if (epause->tx_pause)
2356 lc->requested_fc |= PAUSE_TX;
2357 if (netif_running(dev))
2358 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2359 lc);
2360 return 0;
2361 }
2362
2363 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2364 {
2365 const struct port_info *pi = netdev_priv(dev);
2366 const struct sge *s = &pi->adapter->sge;
2367
2368 e->rx_max_pending = MAX_RX_BUFFERS;
2369 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2370 e->rx_jumbo_max_pending = 0;
2371 e->tx_max_pending = MAX_TXQ_ENTRIES;
2372
2373 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2374 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2375 e->rx_jumbo_pending = 0;
2376 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2377 }
2378
2379 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2380 {
2381 int i;
2382 const struct port_info *pi = netdev_priv(dev);
2383 struct adapter *adapter = pi->adapter;
2384 struct sge *s = &adapter->sge;
2385
2386 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2387 e->tx_pending > MAX_TXQ_ENTRIES ||
2388 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2389 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2390 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2391 return -EINVAL;
2392
2393 if (adapter->flags & FULL_INIT_DONE)
2394 return -EBUSY;
2395
2396 for (i = 0; i < pi->nqsets; ++i) {
2397 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2398 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2399 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2400 }
2401 return 0;
2402 }
2403
2404 static int closest_timer(const struct sge *s, int time)
2405 {
2406 int i, delta, match = 0, min_delta = INT_MAX;
2407
2408 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2409 delta = time - s->timer_val[i];
2410 if (delta < 0)
2411 delta = -delta;
2412 if (delta < min_delta) {
2413 min_delta = delta;
2414 match = i;
2415 }
2416 }
2417 return match;
2418 }
2419
2420 static int closest_thres(const struct sge *s, int thres)
2421 {
2422 int i, delta, match = 0, min_delta = INT_MAX;
2423
2424 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2425 delta = thres - s->counter_val[i];
2426 if (delta < 0)
2427 delta = -delta;
2428 if (delta < min_delta) {
2429 min_delta = delta;
2430 match = i;
2431 }
2432 }
2433 return match;
2434 }
2435
2436 /*
2437 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2438 */
2439 static unsigned int qtimer_val(const struct adapter *adap,
2440 const struct sge_rspq *q)
2441 {
2442 unsigned int idx = q->intr_params >> 1;
2443
2444 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2445 }
2446
2447 /**
2448 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
2449 * @adap: the adapter
2450 * @q: the Rx queue
2451 * @us: the hold-off time in us, or 0 to disable timer
2452 * @cnt: the hold-off packet count, or 0 to disable counter
2453 *
2454 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2455 * one of the two needs to be enabled for the queue to generate interrupts.
2456 */
2457 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
2458 unsigned int us, unsigned int cnt)
2459 {
2460 if ((us | cnt) == 0)
2461 cnt = 1;
2462
2463 if (cnt) {
2464 int err;
2465 u32 v, new_idx;
2466
2467 new_idx = closest_thres(&adap->sge, cnt);
2468 if (q->desc && q->pktcnt_idx != new_idx) {
2469 /* the queue has already been created, update it */
2470 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2471 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2472 FW_PARAMS_PARAM_YZ(q->cntxt_id);
2473 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2474 &new_idx);
2475 if (err)
2476 return err;
2477 }
2478 q->pktcnt_idx = new_idx;
2479 }
2480
2481 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2482 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2483 return 0;
2484 }
2485
2486 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2487 {
2488 const struct port_info *pi = netdev_priv(dev);
2489 struct adapter *adap = pi->adapter;
2490 struct sge_rspq *q;
2491 int i;
2492 int r = 0;
2493
2494 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) {
2495 q = &adap->sge.ethrxq[i].rspq;
2496 r = set_rxq_intr_params(adap, q, c->rx_coalesce_usecs,
2497 c->rx_max_coalesced_frames);
2498 if (r) {
2499 dev_err(&dev->dev, "failed to set coalesce %d\n", r);
2500 break;
2501 }
2502 }
2503 return r;
2504 }
2505
2506 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2507 {
2508 const struct port_info *pi = netdev_priv(dev);
2509 const struct adapter *adap = pi->adapter;
2510 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2511
2512 c->rx_coalesce_usecs = qtimer_val(adap, rq);
2513 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2514 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2515 return 0;
2516 }
2517
2518 /**
2519 * eeprom_ptov - translate a physical EEPROM address to virtual
2520 * @phys_addr: the physical EEPROM address
2521 * @fn: the PCI function number
2522 * @sz: size of function-specific area
2523 *
2524 * Translate a physical EEPROM address to virtual. The first 1K is
2525 * accessed through virtual addresses starting at 31K, the rest is
2526 * accessed through virtual addresses starting at 0.
2527 *
2528 * The mapping is as follows:
2529 * [0..1K) -> [31K..32K)
2530 * [1K..1K+A) -> [31K-A..31K)
2531 * [1K+A..ES) -> [0..ES-A-1K)
2532 *
2533 * where A = @fn * @sz, and ES = EEPROM size.
2534 */
2535 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2536 {
2537 fn *= sz;
2538 if (phys_addr < 1024)
2539 return phys_addr + (31 << 10);
2540 if (phys_addr < 1024 + fn)
2541 return 31744 - fn + phys_addr - 1024;
2542 if (phys_addr < EEPROMSIZE)
2543 return phys_addr - 1024 - fn;
2544 return -EINVAL;
2545 }
2546
2547 /*
2548 * The next two routines implement eeprom read/write from physical addresses.
2549 */
2550 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2551 {
2552 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2553
2554 if (vaddr >= 0)
2555 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2556 return vaddr < 0 ? vaddr : 0;
2557 }
2558
2559 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2560 {
2561 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2562
2563 if (vaddr >= 0)
2564 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2565 return vaddr < 0 ? vaddr : 0;
2566 }
2567
2568 #define EEPROM_MAGIC 0x38E2F10C
2569
2570 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2571 u8 *data)
2572 {
2573 int i, err = 0;
2574 struct adapter *adapter = netdev2adap(dev);
2575
2576 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2577 if (!buf)
2578 return -ENOMEM;
2579
2580 e->magic = EEPROM_MAGIC;
2581 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2582 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2583
2584 if (!err)
2585 memcpy(data, buf + e->offset, e->len);
2586 kfree(buf);
2587 return err;
2588 }
2589
2590 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2591 u8 *data)
2592 {
2593 u8 *buf;
2594 int err = 0;
2595 u32 aligned_offset, aligned_len, *p;
2596 struct adapter *adapter = netdev2adap(dev);
2597
2598 if (eeprom->magic != EEPROM_MAGIC)
2599 return -EINVAL;
2600
2601 aligned_offset = eeprom->offset & ~3;
2602 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2603
2604 if (adapter->fn > 0) {
2605 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2606
2607 if (aligned_offset < start ||
2608 aligned_offset + aligned_len > start + EEPROMPFSIZE)
2609 return -EPERM;
2610 }
2611
2612 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2613 /*
2614 * RMW possibly needed for first or last words.
2615 */
2616 buf = kmalloc(aligned_len, GFP_KERNEL);
2617 if (!buf)
2618 return -ENOMEM;
2619 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2620 if (!err && aligned_len > 4)
2621 err = eeprom_rd_phys(adapter,
2622 aligned_offset + aligned_len - 4,
2623 (u32 *)&buf[aligned_len - 4]);
2624 if (err)
2625 goto out;
2626 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2627 } else
2628 buf = data;
2629
2630 err = t4_seeprom_wp(adapter, false);
2631 if (err)
2632 goto out;
2633
2634 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2635 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2636 aligned_offset += 4;
2637 }
2638
2639 if (!err)
2640 err = t4_seeprom_wp(adapter, true);
2641 out:
2642 if (buf != data)
2643 kfree(buf);
2644 return err;
2645 }
2646
2647 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2648 {
2649 int ret;
2650 const struct firmware *fw;
2651 struct adapter *adap = netdev2adap(netdev);
2652
2653 ef->data[sizeof(ef->data) - 1] = '\0';
2654 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2655 if (ret < 0)
2656 return ret;
2657
2658 ret = t4_load_fw(adap, fw->data, fw->size);
2659 release_firmware(fw);
2660 if (!ret)
2661 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
2662 return ret;
2663 }
2664
2665 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2666 #define BCAST_CRC 0xa0ccc1a6
2667
2668 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2669 {
2670 wol->supported = WAKE_BCAST | WAKE_MAGIC;
2671 wol->wolopts = netdev2adap(dev)->wol;
2672 memset(&wol->sopass, 0, sizeof(wol->sopass));
2673 }
2674
2675 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2676 {
2677 int err = 0;
2678 struct port_info *pi = netdev_priv(dev);
2679
2680 if (wol->wolopts & ~WOL_SUPPORTED)
2681 return -EINVAL;
2682 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2683 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2684 if (wol->wolopts & WAKE_BCAST) {
2685 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2686 ~0ULL, 0, false);
2687 if (!err)
2688 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2689 ~6ULL, ~0ULL, BCAST_CRC, true);
2690 } else
2691 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2692 return err;
2693 }
2694
2695 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2696 {
2697 const struct port_info *pi = netdev_priv(dev);
2698 netdev_features_t changed = dev->features ^ features;
2699 int err;
2700
2701 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2702 return 0;
2703
2704 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2705 -1, -1, -1,
2706 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2707 if (unlikely(err))
2708 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
2709 return err;
2710 }
2711
2712 static u32 get_rss_table_size(struct net_device *dev)
2713 {
2714 const struct port_info *pi = netdev_priv(dev);
2715
2716 return pi->rss_size;
2717 }
2718
2719 static int get_rss_table(struct net_device *dev, u32 *p)
2720 {
2721 const struct port_info *pi = netdev_priv(dev);
2722 unsigned int n = pi->rss_size;
2723
2724 while (n--)
2725 p[n] = pi->rss[n];
2726 return 0;
2727 }
2728
2729 static int set_rss_table(struct net_device *dev, const u32 *p)
2730 {
2731 unsigned int i;
2732 struct port_info *pi = netdev_priv(dev);
2733
2734 for (i = 0; i < pi->rss_size; i++)
2735 pi->rss[i] = p[i];
2736 if (pi->adapter->flags & FULL_INIT_DONE)
2737 return write_rss(pi, pi->rss);
2738 return 0;
2739 }
2740
2741 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
2742 u32 *rules)
2743 {
2744 const struct port_info *pi = netdev_priv(dev);
2745
2746 switch (info->cmd) {
2747 case ETHTOOL_GRXFH: {
2748 unsigned int v = pi->rss_mode;
2749
2750 info->data = 0;
2751 switch (info->flow_type) {
2752 case TCP_V4_FLOW:
2753 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
2754 info->data = RXH_IP_SRC | RXH_IP_DST |
2755 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2756 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2757 info->data = RXH_IP_SRC | RXH_IP_DST;
2758 break;
2759 case UDP_V4_FLOW:
2760 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
2761 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2762 info->data = RXH_IP_SRC | RXH_IP_DST |
2763 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2764 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2765 info->data = RXH_IP_SRC | RXH_IP_DST;
2766 break;
2767 case SCTP_V4_FLOW:
2768 case AH_ESP_V4_FLOW:
2769 case IPV4_FLOW:
2770 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2771 info->data = RXH_IP_SRC | RXH_IP_DST;
2772 break;
2773 case TCP_V6_FLOW:
2774 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
2775 info->data = RXH_IP_SRC | RXH_IP_DST |
2776 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2777 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2778 info->data = RXH_IP_SRC | RXH_IP_DST;
2779 break;
2780 case UDP_V6_FLOW:
2781 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
2782 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2783 info->data = RXH_IP_SRC | RXH_IP_DST |
2784 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2785 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2786 info->data = RXH_IP_SRC | RXH_IP_DST;
2787 break;
2788 case SCTP_V6_FLOW:
2789 case AH_ESP_V6_FLOW:
2790 case IPV6_FLOW:
2791 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2792 info->data = RXH_IP_SRC | RXH_IP_DST;
2793 break;
2794 }
2795 return 0;
2796 }
2797 case ETHTOOL_GRXRINGS:
2798 info->data = pi->nqsets;
2799 return 0;
2800 }
2801 return -EOPNOTSUPP;
2802 }
2803
2804 static const struct ethtool_ops cxgb_ethtool_ops = {
2805 .get_settings = get_settings,
2806 .set_settings = set_settings,
2807 .get_drvinfo = get_drvinfo,
2808 .get_msglevel = get_msglevel,
2809 .set_msglevel = set_msglevel,
2810 .get_ringparam = get_sge_param,
2811 .set_ringparam = set_sge_param,
2812 .get_coalesce = get_coalesce,
2813 .set_coalesce = set_coalesce,
2814 .get_eeprom_len = get_eeprom_len,
2815 .get_eeprom = get_eeprom,
2816 .set_eeprom = set_eeprom,
2817 .get_pauseparam = get_pauseparam,
2818 .set_pauseparam = set_pauseparam,
2819 .get_link = ethtool_op_get_link,
2820 .get_strings = get_strings,
2821 .set_phys_id = identify_port,
2822 .nway_reset = restart_autoneg,
2823 .get_sset_count = get_sset_count,
2824 .get_ethtool_stats = get_stats,
2825 .get_regs_len = get_regs_len,
2826 .get_regs = get_regs,
2827 .get_wol = get_wol,
2828 .set_wol = set_wol,
2829 .get_rxnfc = get_rxnfc,
2830 .get_rxfh_indir_size = get_rss_table_size,
2831 .get_rxfh_indir = get_rss_table,
2832 .set_rxfh_indir = set_rss_table,
2833 .flash_device = set_flash,
2834 };
2835
2836 /*
2837 * debugfs support
2838 */
2839 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2840 loff_t *ppos)
2841 {
2842 loff_t pos = *ppos;
2843 loff_t avail = file_inode(file)->i_size;
2844 unsigned int mem = (uintptr_t)file->private_data & 3;
2845 struct adapter *adap = file->private_data - mem;
2846
2847 if (pos < 0)
2848 return -EINVAL;
2849 if (pos >= avail)
2850 return 0;
2851 if (count > avail - pos)
2852 count = avail - pos;
2853
2854 while (count) {
2855 size_t len;
2856 int ret, ofst;
2857 __be32 data[16];
2858
2859 if ((mem == MEM_MC) || (mem == MEM_MC1))
2860 ret = t4_mc_read(adap, mem % MEM_MC, pos, data, NULL);
2861 else
2862 ret = t4_edc_read(adap, mem, pos, data, NULL);
2863 if (ret)
2864 return ret;
2865
2866 ofst = pos % sizeof(data);
2867 len = min(count, sizeof(data) - ofst);
2868 if (copy_to_user(buf, (u8 *)data + ofst, len))
2869 return -EFAULT;
2870
2871 buf += len;
2872 pos += len;
2873 count -= len;
2874 }
2875 count = pos - *ppos;
2876 *ppos = pos;
2877 return count;
2878 }
2879
2880 static const struct file_operations mem_debugfs_fops = {
2881 .owner = THIS_MODULE,
2882 .open = simple_open,
2883 .read = mem_read,
2884 .llseek = default_llseek,
2885 };
2886
2887 static void add_debugfs_mem(struct adapter *adap, const char *name,
2888 unsigned int idx, unsigned int size_mb)
2889 {
2890 struct dentry *de;
2891
2892 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2893 (void *)adap + idx, &mem_debugfs_fops);
2894 if (de && de->d_inode)
2895 de->d_inode->i_size = size_mb << 20;
2896 }
2897
2898 static int setup_debugfs(struct adapter *adap)
2899 {
2900 int i;
2901 u32 size;
2902
2903 if (IS_ERR_OR_NULL(adap->debugfs_root))
2904 return -1;
2905
2906 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2907 if (i & EDRAM0_ENABLE) {
2908 size = t4_read_reg(adap, MA_EDRAM0_BAR);
2909 add_debugfs_mem(adap, "edc0", MEM_EDC0, EDRAM_SIZE_GET(size));
2910 }
2911 if (i & EDRAM1_ENABLE) {
2912 size = t4_read_reg(adap, MA_EDRAM1_BAR);
2913 add_debugfs_mem(adap, "edc1", MEM_EDC1, EDRAM_SIZE_GET(size));
2914 }
2915 if (is_t4(adap->params.chip)) {
2916 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2917 if (i & EXT_MEM_ENABLE)
2918 add_debugfs_mem(adap, "mc", MEM_MC,
2919 EXT_MEM_SIZE_GET(size));
2920 } else {
2921 if (i & EXT_MEM_ENABLE) {
2922 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2923 add_debugfs_mem(adap, "mc0", MEM_MC0,
2924 EXT_MEM_SIZE_GET(size));
2925 }
2926 if (i & EXT_MEM1_ENABLE) {
2927 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR);
2928 add_debugfs_mem(adap, "mc1", MEM_MC1,
2929 EXT_MEM_SIZE_GET(size));
2930 }
2931 }
2932 if (adap->l2t)
2933 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2934 &t4_l2t_fops);
2935 return 0;
2936 }
2937
2938 /*
2939 * upper-layer driver support
2940 */
2941
2942 /*
2943 * Allocate an active-open TID and set it to the supplied value.
2944 */
2945 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2946 {
2947 int atid = -1;
2948
2949 spin_lock_bh(&t->atid_lock);
2950 if (t->afree) {
2951 union aopen_entry *p = t->afree;
2952
2953 atid = (p - t->atid_tab) + t->atid_base;
2954 t->afree = p->next;
2955 p->data = data;
2956 t->atids_in_use++;
2957 }
2958 spin_unlock_bh(&t->atid_lock);
2959 return atid;
2960 }
2961 EXPORT_SYMBOL(cxgb4_alloc_atid);
2962
2963 /*
2964 * Release an active-open TID.
2965 */
2966 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2967 {
2968 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
2969
2970 spin_lock_bh(&t->atid_lock);
2971 p->next = t->afree;
2972 t->afree = p;
2973 t->atids_in_use--;
2974 spin_unlock_bh(&t->atid_lock);
2975 }
2976 EXPORT_SYMBOL(cxgb4_free_atid);
2977
2978 /*
2979 * Allocate a server TID and set it to the supplied value.
2980 */
2981 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2982 {
2983 int stid;
2984
2985 spin_lock_bh(&t->stid_lock);
2986 if (family == PF_INET) {
2987 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2988 if (stid < t->nstids)
2989 __set_bit(stid, t->stid_bmap);
2990 else
2991 stid = -1;
2992 } else {
2993 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2994 if (stid < 0)
2995 stid = -1;
2996 }
2997 if (stid >= 0) {
2998 t->stid_tab[stid].data = data;
2999 stid += t->stid_base;
3000 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3001 * This is equivalent to 4 TIDs. With CLIP enabled it
3002 * needs 2 TIDs.
3003 */
3004 if (family == PF_INET)
3005 t->stids_in_use++;
3006 else
3007 t->stids_in_use += 4;
3008 }
3009 spin_unlock_bh(&t->stid_lock);
3010 return stid;
3011 }
3012 EXPORT_SYMBOL(cxgb4_alloc_stid);
3013
3014 /* Allocate a server filter TID and set it to the supplied value.
3015 */
3016 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3017 {
3018 int stid;
3019
3020 spin_lock_bh(&t->stid_lock);
3021 if (family == PF_INET) {
3022 stid = find_next_zero_bit(t->stid_bmap,
3023 t->nstids + t->nsftids, t->nstids);
3024 if (stid < (t->nstids + t->nsftids))
3025 __set_bit(stid, t->stid_bmap);
3026 else
3027 stid = -1;
3028 } else {
3029 stid = -1;
3030 }
3031 if (stid >= 0) {
3032 t->stid_tab[stid].data = data;
3033 stid -= t->nstids;
3034 stid += t->sftid_base;
3035 t->stids_in_use++;
3036 }
3037 spin_unlock_bh(&t->stid_lock);
3038 return stid;
3039 }
3040 EXPORT_SYMBOL(cxgb4_alloc_sftid);
3041
3042 /* Release a server TID.
3043 */
3044 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3045 {
3046 /* Is it a server filter TID? */
3047 if (t->nsftids && (stid >= t->sftid_base)) {
3048 stid -= t->sftid_base;
3049 stid += t->nstids;
3050 } else {
3051 stid -= t->stid_base;
3052 }
3053
3054 spin_lock_bh(&t->stid_lock);
3055 if (family == PF_INET)
3056 __clear_bit(stid, t->stid_bmap);
3057 else
3058 bitmap_release_region(t->stid_bmap, stid, 2);
3059 t->stid_tab[stid].data = NULL;
3060 if (family == PF_INET)
3061 t->stids_in_use--;
3062 else
3063 t->stids_in_use -= 4;
3064 spin_unlock_bh(&t->stid_lock);
3065 }
3066 EXPORT_SYMBOL(cxgb4_free_stid);
3067
3068 /*
3069 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3070 */
3071 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3072 unsigned int tid)
3073 {
3074 struct cpl_tid_release *req;
3075
3076 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3077 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3078 INIT_TP_WR(req, tid);
3079 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3080 }
3081
3082 /*
3083 * Queue a TID release request and if necessary schedule a work queue to
3084 * process it.
3085 */
3086 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3087 unsigned int tid)
3088 {
3089 void **p = &t->tid_tab[tid];
3090 struct adapter *adap = container_of(t, struct adapter, tids);
3091
3092 spin_lock_bh(&adap->tid_release_lock);
3093 *p = adap->tid_release_head;
3094 /* Low 2 bits encode the Tx channel number */
3095 adap->tid_release_head = (void **)((uintptr_t)p | chan);
3096 if (!adap->tid_release_task_busy) {
3097 adap->tid_release_task_busy = true;
3098 queue_work(workq, &adap->tid_release_task);
3099 }
3100 spin_unlock_bh(&adap->tid_release_lock);
3101 }
3102
3103 /*
3104 * Process the list of pending TID release requests.
3105 */
3106 static void process_tid_release_list(struct work_struct *work)
3107 {
3108 struct sk_buff *skb;
3109 struct adapter *adap;
3110
3111 adap = container_of(work, struct adapter, tid_release_task);
3112
3113 spin_lock_bh(&adap->tid_release_lock);
3114 while (adap->tid_release_head) {
3115 void **p = adap->tid_release_head;
3116 unsigned int chan = (uintptr_t)p & 3;
3117 p = (void *)p - chan;
3118
3119 adap->tid_release_head = *p;
3120 *p = NULL;
3121 spin_unlock_bh(&adap->tid_release_lock);
3122
3123 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3124 GFP_KERNEL)))
3125 schedule_timeout_uninterruptible(1);
3126
3127 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3128 t4_ofld_send(adap, skb);
3129 spin_lock_bh(&adap->tid_release_lock);
3130 }
3131 adap->tid_release_task_busy = false;
3132 spin_unlock_bh(&adap->tid_release_lock);
3133 }
3134
3135 /*
3136 * Release a TID and inform HW. If we are unable to allocate the release
3137 * message we defer to a work queue.
3138 */
3139 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3140 {
3141 void *old;
3142 struct sk_buff *skb;
3143 struct adapter *adap = container_of(t, struct adapter, tids);
3144
3145 old = t->tid_tab[tid];
3146 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3147 if (likely(skb)) {
3148 t->tid_tab[tid] = NULL;
3149 mk_tid_release(skb, chan, tid);
3150 t4_ofld_send(adap, skb);
3151 } else
3152 cxgb4_queue_tid_release(t, chan, tid);
3153 if (old)
3154 atomic_dec(&t->tids_in_use);
3155 }
3156 EXPORT_SYMBOL(cxgb4_remove_tid);
3157
3158 /*
3159 * Allocate and initialize the TID tables. Returns 0 on success.
3160 */
3161 static int tid_init(struct tid_info *t)
3162 {
3163 size_t size;
3164 unsigned int stid_bmap_size;
3165 unsigned int natids = t->natids;
3166 struct adapter *adap = container_of(t, struct adapter, tids);
3167
3168 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
3169 size = t->ntids * sizeof(*t->tid_tab) +
3170 natids * sizeof(*t->atid_tab) +
3171 t->nstids * sizeof(*t->stid_tab) +
3172 t->nsftids * sizeof(*t->stid_tab) +
3173 stid_bmap_size * sizeof(long) +
3174 t->nftids * sizeof(*t->ftid_tab) +
3175 t->nsftids * sizeof(*t->ftid_tab);
3176
3177 t->tid_tab = t4_alloc_mem(size);
3178 if (!t->tid_tab)
3179 return -ENOMEM;
3180
3181 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3182 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
3183 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
3184 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
3185 spin_lock_init(&t->stid_lock);
3186 spin_lock_init(&t->atid_lock);
3187
3188 t->stids_in_use = 0;
3189 t->afree = NULL;
3190 t->atids_in_use = 0;
3191 atomic_set(&t->tids_in_use, 0);
3192
3193 /* Setup the free list for atid_tab and clear the stid bitmap. */
3194 if (natids) {
3195 while (--natids)
3196 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3197 t->afree = t->atid_tab;
3198 }
3199 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
3200 /* Reserve stid 0 for T4/T5 adapters */
3201 if (!t->stid_base &&
3202 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3203 __set_bit(0, t->stid_bmap);
3204
3205 return 0;
3206 }
3207
3208 static int cxgb4_clip_get(const struct net_device *dev,
3209 const struct in6_addr *lip)
3210 {
3211 struct adapter *adap;
3212 struct fw_clip_cmd c;
3213
3214 adap = netdev2adap(dev);
3215 memset(&c, 0, sizeof(c));
3216 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3217 FW_CMD_REQUEST | FW_CMD_WRITE);
3218 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c));
3219 *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr);
3220 *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3221 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3222 }
3223
3224 static int cxgb4_clip_release(const struct net_device *dev,
3225 const struct in6_addr *lip)
3226 {
3227 struct adapter *adap;
3228 struct fw_clip_cmd c;
3229
3230 adap = netdev2adap(dev);
3231 memset(&c, 0, sizeof(c));
3232 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3233 FW_CMD_REQUEST | FW_CMD_READ);
3234 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c));
3235 *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr);
3236 *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3237 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3238 }
3239
3240 /**
3241 * cxgb4_create_server - create an IP server
3242 * @dev: the device
3243 * @stid: the server TID
3244 * @sip: local IP address to bind server to
3245 * @sport: the server's TCP port
3246 * @queue: queue to direct messages from this server to
3247 *
3248 * Create an IP server for the given port and address.
3249 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3250 */
3251 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
3252 __be32 sip, __be16 sport, __be16 vlan,
3253 unsigned int queue)
3254 {
3255 unsigned int chan;
3256 struct sk_buff *skb;
3257 struct adapter *adap;
3258 struct cpl_pass_open_req *req;
3259 int ret;
3260
3261 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3262 if (!skb)
3263 return -ENOMEM;
3264
3265 adap = netdev2adap(dev);
3266 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3267 INIT_TP_WR(req, 0);
3268 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3269 req->local_port = sport;
3270 req->peer_port = htons(0);
3271 req->local_ip = sip;
3272 req->peer_ip = htonl(0);
3273 chan = rxq_to_chan(&adap->sge, queue);
3274 req->opt0 = cpu_to_be64(TX_CHAN(chan));
3275 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3276 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3277 ret = t4_mgmt_tx(adap, skb);
3278 return net_xmit_eval(ret);
3279 }
3280 EXPORT_SYMBOL(cxgb4_create_server);
3281
3282 /* cxgb4_create_server6 - create an IPv6 server
3283 * @dev: the device
3284 * @stid: the server TID
3285 * @sip: local IPv6 address to bind server to
3286 * @sport: the server's TCP port
3287 * @queue: queue to direct messages from this server to
3288 *
3289 * Create an IPv6 server for the given port and address.
3290 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3291 */
3292 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3293 const struct in6_addr *sip, __be16 sport,
3294 unsigned int queue)
3295 {
3296 unsigned int chan;
3297 struct sk_buff *skb;
3298 struct adapter *adap;
3299 struct cpl_pass_open_req6 *req;
3300 int ret;
3301
3302 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3303 if (!skb)
3304 return -ENOMEM;
3305
3306 adap = netdev2adap(dev);
3307 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3308 INIT_TP_WR(req, 0);
3309 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3310 req->local_port = sport;
3311 req->peer_port = htons(0);
3312 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3313 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3314 req->peer_ip_hi = cpu_to_be64(0);
3315 req->peer_ip_lo = cpu_to_be64(0);
3316 chan = rxq_to_chan(&adap->sge, queue);
3317 req->opt0 = cpu_to_be64(TX_CHAN(chan));
3318 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3319 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3320 ret = t4_mgmt_tx(adap, skb);
3321 return net_xmit_eval(ret);
3322 }
3323 EXPORT_SYMBOL(cxgb4_create_server6);
3324
3325 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3326 unsigned int queue, bool ipv6)
3327 {
3328 struct sk_buff *skb;
3329 struct adapter *adap;
3330 struct cpl_close_listsvr_req *req;
3331 int ret;
3332
3333 adap = netdev2adap(dev);
3334
3335 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3336 if (!skb)
3337 return -ENOMEM;
3338
3339 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3340 INIT_TP_WR(req, 0);
3341 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3342 req->reply_ctrl = htons(NO_REPLY(0) | (ipv6 ? LISTSVR_IPV6(1) :
3343 LISTSVR_IPV6(0)) | QUEUENO(queue));
3344 ret = t4_mgmt_tx(adap, skb);
3345 return net_xmit_eval(ret);
3346 }
3347 EXPORT_SYMBOL(cxgb4_remove_server);
3348
3349 /**
3350 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3351 * @mtus: the HW MTU table
3352 * @mtu: the target MTU
3353 * @idx: index of selected entry in the MTU table
3354 *
3355 * Returns the index and the value in the HW MTU table that is closest to
3356 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3357 * table, in which case that smallest available value is selected.
3358 */
3359 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3360 unsigned int *idx)
3361 {
3362 unsigned int i = 0;
3363
3364 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3365 ++i;
3366 if (idx)
3367 *idx = i;
3368 return mtus[i];
3369 }
3370 EXPORT_SYMBOL(cxgb4_best_mtu);
3371
3372 /**
3373 * cxgb4_port_chan - get the HW channel of a port
3374 * @dev: the net device for the port
3375 *
3376 * Return the HW Tx channel of the given port.
3377 */
3378 unsigned int cxgb4_port_chan(const struct net_device *dev)
3379 {
3380 return netdev2pinfo(dev)->tx_chan;
3381 }
3382 EXPORT_SYMBOL(cxgb4_port_chan);
3383
3384 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3385 {
3386 struct adapter *adap = netdev2adap(dev);
3387 u32 v1, v2, lp_count, hp_count;
3388
3389 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3390 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
3391 if (is_t4(adap->params.chip)) {
3392 lp_count = G_LP_COUNT(v1);
3393 hp_count = G_HP_COUNT(v1);
3394 } else {
3395 lp_count = G_LP_COUNT_T5(v1);
3396 hp_count = G_HP_COUNT_T5(v2);
3397 }
3398 return lpfifo ? lp_count : hp_count;
3399 }
3400 EXPORT_SYMBOL(cxgb4_dbfifo_count);
3401
3402 /**
3403 * cxgb4_port_viid - get the VI id of a port
3404 * @dev: the net device for the port
3405 *
3406 * Return the VI id of the given port.
3407 */
3408 unsigned int cxgb4_port_viid(const struct net_device *dev)
3409 {
3410 return netdev2pinfo(dev)->viid;
3411 }
3412 EXPORT_SYMBOL(cxgb4_port_viid);
3413
3414 /**
3415 * cxgb4_port_idx - get the index of a port
3416 * @dev: the net device for the port
3417 *
3418 * Return the index of the given port.
3419 */
3420 unsigned int cxgb4_port_idx(const struct net_device *dev)
3421 {
3422 return netdev2pinfo(dev)->port_id;
3423 }
3424 EXPORT_SYMBOL(cxgb4_port_idx);
3425
3426 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3427 struct tp_tcp_stats *v6)
3428 {
3429 struct adapter *adap = pci_get_drvdata(pdev);
3430
3431 spin_lock(&adap->stats_lock);
3432 t4_tp_get_tcp_stats(adap, v4, v6);
3433 spin_unlock(&adap->stats_lock);
3434 }
3435 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3436
3437 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3438 const unsigned int *pgsz_order)
3439 {
3440 struct adapter *adap = netdev2adap(dev);
3441
3442 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
3443 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
3444 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
3445 HPZ3(pgsz_order[3]));
3446 }
3447 EXPORT_SYMBOL(cxgb4_iscsi_init);
3448
3449 int cxgb4_flush_eq_cache(struct net_device *dev)
3450 {
3451 struct adapter *adap = netdev2adap(dev);
3452 int ret;
3453
3454 ret = t4_fwaddrspace_write(adap, adap->mbox,
3455 0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
3456 return ret;
3457 }
3458 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3459
3460 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3461 {
3462 u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
3463 __be64 indices;
3464 int ret;
3465
3466 ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
3467 if (!ret) {
3468 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3469 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3470 }
3471 return ret;
3472 }
3473
3474 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3475 u16 size)
3476 {
3477 struct adapter *adap = netdev2adap(dev);
3478 u16 hw_pidx, hw_cidx;
3479 int ret;
3480
3481 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3482 if (ret)
3483 goto out;
3484
3485 if (pidx != hw_pidx) {
3486 u16 delta;
3487
3488 if (pidx >= hw_pidx)
3489 delta = pidx - hw_pidx;
3490 else
3491 delta = size - hw_pidx + pidx;
3492 wmb();
3493 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3494 QID(qid) | PIDX(delta));
3495 }
3496 out:
3497 return ret;
3498 }
3499 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3500
3501 void cxgb4_disable_db_coalescing(struct net_device *dev)
3502 {
3503 struct adapter *adap;
3504
3505 adap = netdev2adap(dev);
3506 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE,
3507 F_NOCOALESCE);
3508 }
3509 EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3510
3511 void cxgb4_enable_db_coalescing(struct net_device *dev)
3512 {
3513 struct adapter *adap;
3514
3515 adap = netdev2adap(dev);
3516 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 0);
3517 }
3518 EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3519
3520 static struct pci_driver cxgb4_driver;
3521
3522 static void check_neigh_update(struct neighbour *neigh)
3523 {
3524 const struct device *parent;
3525 const struct net_device *netdev = neigh->dev;
3526
3527 if (netdev->priv_flags & IFF_802_1Q_VLAN)
3528 netdev = vlan_dev_real_dev(netdev);
3529 parent = netdev->dev.parent;
3530 if (parent && parent->driver == &cxgb4_driver.driver)
3531 t4_l2t_update(dev_get_drvdata(parent), neigh);
3532 }
3533
3534 static int netevent_cb(struct notifier_block *nb, unsigned long event,
3535 void *data)
3536 {
3537 switch (event) {
3538 case NETEVENT_NEIGH_UPDATE:
3539 check_neigh_update(data);
3540 break;
3541 case NETEVENT_REDIRECT:
3542 default:
3543 break;
3544 }
3545 return 0;
3546 }
3547
3548 static bool netevent_registered;
3549 static struct notifier_block cxgb4_netevent_nb = {
3550 .notifier_call = netevent_cb
3551 };
3552
3553 static void drain_db_fifo(struct adapter *adap, int usecs)
3554 {
3555 u32 v1, v2, lp_count, hp_count;
3556
3557 do {
3558 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3559 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
3560 if (is_t4(adap->params.chip)) {
3561 lp_count = G_LP_COUNT(v1);
3562 hp_count = G_HP_COUNT(v1);
3563 } else {
3564 lp_count = G_LP_COUNT_T5(v1);
3565 hp_count = G_HP_COUNT_T5(v2);
3566 }
3567
3568 if (lp_count == 0 && hp_count == 0)
3569 break;
3570 set_current_state(TASK_UNINTERRUPTIBLE);
3571 schedule_timeout(usecs_to_jiffies(usecs));
3572 } while (1);
3573 }
3574
3575 static void disable_txq_db(struct sge_txq *q)
3576 {
3577 spin_lock_irq(&q->db_lock);
3578 q->db_disabled = 1;
3579 spin_unlock_irq(&q->db_lock);
3580 }
3581
3582 static void enable_txq_db(struct sge_txq *q)
3583 {
3584 spin_lock_irq(&q->db_lock);
3585 q->db_disabled = 0;
3586 spin_unlock_irq(&q->db_lock);
3587 }
3588
3589 static void disable_dbs(struct adapter *adap)
3590 {
3591 int i;
3592
3593 for_each_ethrxq(&adap->sge, i)
3594 disable_txq_db(&adap->sge.ethtxq[i].q);
3595 for_each_ofldrxq(&adap->sge, i)
3596 disable_txq_db(&adap->sge.ofldtxq[i].q);
3597 for_each_port(adap, i)
3598 disable_txq_db(&adap->sge.ctrlq[i].q);
3599 }
3600
3601 static void enable_dbs(struct adapter *adap)
3602 {
3603 int i;
3604
3605 for_each_ethrxq(&adap->sge, i)
3606 enable_txq_db(&adap->sge.ethtxq[i].q);
3607 for_each_ofldrxq(&adap->sge, i)
3608 enable_txq_db(&adap->sge.ofldtxq[i].q);
3609 for_each_port(adap, i)
3610 enable_txq_db(&adap->sge.ctrlq[i].q);
3611 }
3612
3613 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3614 {
3615 u16 hw_pidx, hw_cidx;
3616 int ret;
3617
3618 spin_lock_bh(&q->db_lock);
3619 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3620 if (ret)
3621 goto out;
3622 if (q->db_pidx != hw_pidx) {
3623 u16 delta;
3624
3625 if (q->db_pidx >= hw_pidx)
3626 delta = q->db_pidx - hw_pidx;
3627 else
3628 delta = q->size - hw_pidx + q->db_pidx;
3629 wmb();
3630 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3631 QID(q->cntxt_id) | PIDX(delta));
3632 }
3633 out:
3634 q->db_disabled = 0;
3635 spin_unlock_bh(&q->db_lock);
3636 if (ret)
3637 CH_WARN(adap, "DB drop recovery failed.\n");
3638 }
3639 static void recover_all_queues(struct adapter *adap)
3640 {
3641 int i;
3642
3643 for_each_ethrxq(&adap->sge, i)
3644 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3645 for_each_ofldrxq(&adap->sge, i)
3646 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3647 for_each_port(adap, i)
3648 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3649 }
3650
3651 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3652 {
3653 mutex_lock(&uld_mutex);
3654 if (adap->uld_handle[CXGB4_ULD_RDMA])
3655 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3656 cmd);
3657 mutex_unlock(&uld_mutex);
3658 }
3659
3660 static void process_db_full(struct work_struct *work)
3661 {
3662 struct adapter *adap;
3663
3664 adap = container_of(work, struct adapter, db_full_task);
3665
3666 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3667 drain_db_fifo(adap, dbfifo_drain_delay);
3668 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3669 DBFIFO_HP_INT | DBFIFO_LP_INT,
3670 DBFIFO_HP_INT | DBFIFO_LP_INT);
3671 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3672 }
3673
3674 static void process_db_drop(struct work_struct *work)
3675 {
3676 struct adapter *adap;
3677
3678 adap = container_of(work, struct adapter, db_drop_task);
3679
3680 if (is_t4(adap->params.chip)) {
3681 disable_dbs(adap);
3682 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
3683 drain_db_fifo(adap, 1);
3684 recover_all_queues(adap);
3685 enable_dbs(adap);
3686 } else {
3687 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3688 u16 qid = (dropped_db >> 15) & 0x1ffff;
3689 u16 pidx_inc = dropped_db & 0x1fff;
3690 unsigned int s_qpp;
3691 unsigned short udb_density;
3692 unsigned long qpshift;
3693 int page;
3694 u32 udb;
3695
3696 dev_warn(adap->pdev_dev,
3697 "Dropped DB 0x%x qid %d bar2 %d coalesce %d pidx %d\n",
3698 dropped_db, qid,
3699 (dropped_db >> 14) & 1,
3700 (dropped_db >> 13) & 1,
3701 pidx_inc);
3702
3703 drain_db_fifo(adap, 1);
3704
3705 s_qpp = QUEUESPERPAGEPF1 * adap->fn;
3706 udb_density = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adap,
3707 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
3708 qpshift = PAGE_SHIFT - ilog2(udb_density);
3709 udb = qid << qpshift;
3710 udb &= PAGE_MASK;
3711 page = udb / PAGE_SIZE;
3712 udb += (qid - (page * udb_density)) * 128;
3713
3714 writel(PIDX(pidx_inc), adap->bar2 + udb + 8);
3715
3716 /* Re-enable BAR2 WC */
3717 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3718 }
3719
3720 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
3721 }
3722
3723 void t4_db_full(struct adapter *adap)
3724 {
3725 if (is_t4(adap->params.chip)) {
3726 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3727 DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
3728 queue_work(workq, &adap->db_full_task);
3729 }
3730 }
3731
3732 void t4_db_dropped(struct adapter *adap)
3733 {
3734 if (is_t4(adap->params.chip))
3735 queue_work(workq, &adap->db_drop_task);
3736 }
3737
3738 static void uld_attach(struct adapter *adap, unsigned int uld)
3739 {
3740 void *handle;
3741 struct cxgb4_lld_info lli;
3742 unsigned short i;
3743
3744 lli.pdev = adap->pdev;
3745 lli.l2t = adap->l2t;
3746 lli.tids = &adap->tids;
3747 lli.ports = adap->port;
3748 lli.vr = &adap->vres;
3749 lli.mtus = adap->params.mtus;
3750 if (uld == CXGB4_ULD_RDMA) {
3751 lli.rxq_ids = adap->sge.rdma_rxq;
3752 lli.nrxq = adap->sge.rdmaqs;
3753 } else if (uld == CXGB4_ULD_ISCSI) {
3754 lli.rxq_ids = adap->sge.ofld_rxq;
3755 lli.nrxq = adap->sge.ofldqsets;
3756 }
3757 lli.ntxq = adap->sge.ofldqsets;
3758 lli.nchan = adap->params.nports;
3759 lli.nports = adap->params.nports;
3760 lli.wr_cred = adap->params.ofldq_wr_cred;
3761 lli.adapter_type = adap->params.chip;
3762 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
3763 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
3764 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
3765 (adap->fn * 4));
3766 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
3767 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
3768 (adap->fn * 4));
3769 lli.filt_mode = adap->params.tp.vlan_pri_map;
3770 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3771 for (i = 0; i < NCHAN; i++)
3772 lli.tx_modq[i] = i;
3773 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
3774 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
3775 lli.fw_vers = adap->params.fw_vers;
3776 lli.dbfifo_int_thresh = dbfifo_int_thresh;
3777 lli.sge_pktshift = adap->sge.pktshift;
3778 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
3779
3780 handle = ulds[uld].add(&lli);
3781 if (IS_ERR(handle)) {
3782 dev_warn(adap->pdev_dev,
3783 "could not attach to the %s driver, error %ld\n",
3784 uld_str[uld], PTR_ERR(handle));
3785 return;
3786 }
3787
3788 adap->uld_handle[uld] = handle;
3789
3790 if (!netevent_registered) {
3791 register_netevent_notifier(&cxgb4_netevent_nb);
3792 netevent_registered = true;
3793 }
3794
3795 if (adap->flags & FULL_INIT_DONE)
3796 ulds[uld].state_change(handle, CXGB4_STATE_UP);
3797 }
3798
3799 static void attach_ulds(struct adapter *adap)
3800 {
3801 unsigned int i;
3802
3803 spin_lock(&adap_rcu_lock);
3804 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
3805 spin_unlock(&adap_rcu_lock);
3806
3807 mutex_lock(&uld_mutex);
3808 list_add_tail(&adap->list_node, &adapter_list);
3809 for (i = 0; i < CXGB4_ULD_MAX; i++)
3810 if (ulds[i].add)
3811 uld_attach(adap, i);
3812 mutex_unlock(&uld_mutex);
3813 }
3814
3815 static void detach_ulds(struct adapter *adap)
3816 {
3817 unsigned int i;
3818
3819 mutex_lock(&uld_mutex);
3820 list_del(&adap->list_node);
3821 for (i = 0; i < CXGB4_ULD_MAX; i++)
3822 if (adap->uld_handle[i]) {
3823 ulds[i].state_change(adap->uld_handle[i],
3824 CXGB4_STATE_DETACH);
3825 adap->uld_handle[i] = NULL;
3826 }
3827 if (netevent_registered && list_empty(&adapter_list)) {
3828 unregister_netevent_notifier(&cxgb4_netevent_nb);
3829 netevent_registered = false;
3830 }
3831 mutex_unlock(&uld_mutex);
3832
3833 spin_lock(&adap_rcu_lock);
3834 list_del_rcu(&adap->rcu_node);
3835 spin_unlock(&adap_rcu_lock);
3836 }
3837
3838 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
3839 {
3840 unsigned int i;
3841
3842 mutex_lock(&uld_mutex);
3843 for (i = 0; i < CXGB4_ULD_MAX; i++)
3844 if (adap->uld_handle[i])
3845 ulds[i].state_change(adap->uld_handle[i], new_state);
3846 mutex_unlock(&uld_mutex);
3847 }
3848
3849 /**
3850 * cxgb4_register_uld - register an upper-layer driver
3851 * @type: the ULD type
3852 * @p: the ULD methods
3853 *
3854 * Registers an upper-layer driver with this driver and notifies the ULD
3855 * about any presently available devices that support its type. Returns
3856 * %-EBUSY if a ULD of the same type is already registered.
3857 */
3858 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
3859 {
3860 int ret = 0;
3861 struct adapter *adap;
3862
3863 if (type >= CXGB4_ULD_MAX)
3864 return -EINVAL;
3865 mutex_lock(&uld_mutex);
3866 if (ulds[type].add) {
3867 ret = -EBUSY;
3868 goto out;
3869 }
3870 ulds[type] = *p;
3871 list_for_each_entry(adap, &adapter_list, list_node)
3872 uld_attach(adap, type);
3873 out: mutex_unlock(&uld_mutex);
3874 return ret;
3875 }
3876 EXPORT_SYMBOL(cxgb4_register_uld);
3877
3878 /**
3879 * cxgb4_unregister_uld - unregister an upper-layer driver
3880 * @type: the ULD type
3881 *
3882 * Unregisters an existing upper-layer driver.
3883 */
3884 int cxgb4_unregister_uld(enum cxgb4_uld type)
3885 {
3886 struct adapter *adap;
3887
3888 if (type >= CXGB4_ULD_MAX)
3889 return -EINVAL;
3890 mutex_lock(&uld_mutex);
3891 list_for_each_entry(adap, &adapter_list, list_node)
3892 adap->uld_handle[type] = NULL;
3893 ulds[type].add = NULL;
3894 mutex_unlock(&uld_mutex);
3895 return 0;
3896 }
3897 EXPORT_SYMBOL(cxgb4_unregister_uld);
3898
3899 /* Check if netdev on which event is occured belongs to us or not. Return
3900 * suceess (1) if it belongs otherwise failure (0).
3901 */
3902 static int cxgb4_netdev(struct net_device *netdev)
3903 {
3904 struct adapter *adap;
3905 int i;
3906
3907 spin_lock(&adap_rcu_lock);
3908 list_for_each_entry_rcu(adap, &adap_rcu_list, rcu_node)
3909 for (i = 0; i < MAX_NPORTS; i++)
3910 if (adap->port[i] == netdev) {
3911 spin_unlock(&adap_rcu_lock);
3912 return 1;
3913 }
3914 spin_unlock(&adap_rcu_lock);
3915 return 0;
3916 }
3917
3918 static int clip_add(struct net_device *event_dev, struct inet6_ifaddr *ifa,
3919 unsigned long event)
3920 {
3921 int ret = NOTIFY_DONE;
3922
3923 rcu_read_lock();
3924 if (cxgb4_netdev(event_dev)) {
3925 switch (event) {
3926 case NETDEV_UP:
3927 ret = cxgb4_clip_get(event_dev,
3928 (const struct in6_addr *)ifa->addr.s6_addr);
3929 if (ret < 0) {
3930 rcu_read_unlock();
3931 return ret;
3932 }
3933 ret = NOTIFY_OK;
3934 break;
3935 case NETDEV_DOWN:
3936 cxgb4_clip_release(event_dev,
3937 (const struct in6_addr *)ifa->addr.s6_addr);
3938 ret = NOTIFY_OK;
3939 break;
3940 default:
3941 break;
3942 }
3943 }
3944 rcu_read_unlock();
3945 return ret;
3946 }
3947
3948 static int cxgb4_inet6addr_handler(struct notifier_block *this,
3949 unsigned long event, void *data)
3950 {
3951 struct inet6_ifaddr *ifa = data;
3952 struct net_device *event_dev;
3953 int ret = NOTIFY_DONE;
3954 struct bonding *bond = netdev_priv(ifa->idev->dev);
3955 struct list_head *iter;
3956 struct slave *slave;
3957 struct pci_dev *first_pdev = NULL;
3958
3959 if (ifa->idev->dev->priv_flags & IFF_802_1Q_VLAN) {
3960 event_dev = vlan_dev_real_dev(ifa->idev->dev);
3961 ret = clip_add(event_dev, ifa, event);
3962 } else if (ifa->idev->dev->flags & IFF_MASTER) {
3963 /* It is possible that two different adapters are bonded in one
3964 * bond. We need to find such different adapters and add clip
3965 * in all of them only once.
3966 */
3967 read_lock(&bond->lock);
3968 bond_for_each_slave(bond, slave, iter) {
3969 if (!first_pdev) {
3970 ret = clip_add(slave->dev, ifa, event);
3971 /* If clip_add is success then only initialize
3972 * first_pdev since it means it is our device
3973 */
3974 if (ret == NOTIFY_OK)
3975 first_pdev = to_pci_dev(
3976 slave->dev->dev.parent);
3977 } else if (first_pdev !=
3978 to_pci_dev(slave->dev->dev.parent))
3979 ret = clip_add(slave->dev, ifa, event);
3980 }
3981 read_unlock(&bond->lock);
3982 } else
3983 ret = clip_add(ifa->idev->dev, ifa, event);
3984
3985 return ret;
3986 }
3987
3988 static struct notifier_block cxgb4_inet6addr_notifier = {
3989 .notifier_call = cxgb4_inet6addr_handler
3990 };
3991
3992 /* Retrieves IPv6 addresses from a root device (bond, vlan) associated with
3993 * a physical device.
3994 * The physical device reference is needed to send the actul CLIP command.
3995 */
3996 static int update_dev_clip(struct net_device *root_dev, struct net_device *dev)
3997 {
3998 struct inet6_dev *idev = NULL;
3999 struct inet6_ifaddr *ifa;
4000 int ret = 0;
4001
4002 idev = __in6_dev_get(root_dev);
4003 if (!idev)
4004 return ret;
4005
4006 read_lock_bh(&idev->lock);
4007 list_for_each_entry(ifa, &idev->addr_list, if_list) {
4008 ret = cxgb4_clip_get(dev,
4009 (const struct in6_addr *)ifa->addr.s6_addr);
4010 if (ret < 0)
4011 break;
4012 }
4013 read_unlock_bh(&idev->lock);
4014
4015 return ret;
4016 }
4017
4018 static int update_root_dev_clip(struct net_device *dev)
4019 {
4020 struct net_device *root_dev = NULL;
4021 int i, ret = 0;
4022
4023 /* First populate the real net device's IPv6 addresses */
4024 ret = update_dev_clip(dev, dev);
4025 if (ret)
4026 return ret;
4027
4028 /* Parse all bond and vlan devices layered on top of the physical dev */
4029 for (i = 0; i < VLAN_N_VID; i++) {
4030 root_dev = __vlan_find_dev_deep(dev, htons(ETH_P_8021Q), i);
4031 if (!root_dev)
4032 continue;
4033
4034 ret = update_dev_clip(root_dev, dev);
4035 if (ret)
4036 break;
4037 }
4038 return ret;
4039 }
4040
4041 static void update_clip(const struct adapter *adap)
4042 {
4043 int i;
4044 struct net_device *dev;
4045 int ret;
4046
4047 rcu_read_lock();
4048
4049 for (i = 0; i < MAX_NPORTS; i++) {
4050 dev = adap->port[i];
4051 ret = 0;
4052
4053 if (dev)
4054 ret = update_root_dev_clip(dev);
4055
4056 if (ret < 0)
4057 break;
4058 }
4059 rcu_read_unlock();
4060 }
4061
4062 /**
4063 * cxgb_up - enable the adapter
4064 * @adap: adapter being enabled
4065 *
4066 * Called when the first port is enabled, this function performs the
4067 * actions necessary to make an adapter operational, such as completing
4068 * the initialization of HW modules, and enabling interrupts.
4069 *
4070 * Must be called with the rtnl lock held.
4071 */
4072 static int cxgb_up(struct adapter *adap)
4073 {
4074 int err;
4075
4076 err = setup_sge_queues(adap);
4077 if (err)
4078 goto out;
4079 err = setup_rss(adap);
4080 if (err)
4081 goto freeq;
4082
4083 if (adap->flags & USING_MSIX) {
4084 name_msix_vecs(adap);
4085 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4086 adap->msix_info[0].desc, adap);
4087 if (err)
4088 goto irq_err;
4089
4090 err = request_msix_queue_irqs(adap);
4091 if (err) {
4092 free_irq(adap->msix_info[0].vec, adap);
4093 goto irq_err;
4094 }
4095 } else {
4096 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4097 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
4098 adap->port[0]->name, adap);
4099 if (err)
4100 goto irq_err;
4101 }
4102 enable_rx(adap);
4103 t4_sge_start(adap);
4104 t4_intr_enable(adap);
4105 adap->flags |= FULL_INIT_DONE;
4106 notify_ulds(adap, CXGB4_STATE_UP);
4107 update_clip(adap);
4108 out:
4109 return err;
4110 irq_err:
4111 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
4112 freeq:
4113 t4_free_sge_resources(adap);
4114 goto out;
4115 }
4116
4117 static void cxgb_down(struct adapter *adapter)
4118 {
4119 t4_intr_disable(adapter);
4120 cancel_work_sync(&adapter->tid_release_task);
4121 cancel_work_sync(&adapter->db_full_task);
4122 cancel_work_sync(&adapter->db_drop_task);
4123 adapter->tid_release_task_busy = false;
4124 adapter->tid_release_head = NULL;
4125
4126 if (adapter->flags & USING_MSIX) {
4127 free_msix_queue_irqs(adapter);
4128 free_irq(adapter->msix_info[0].vec, adapter);
4129 } else
4130 free_irq(adapter->pdev->irq, adapter);
4131 quiesce_rx(adapter);
4132 t4_sge_stop(adapter);
4133 t4_free_sge_resources(adapter);
4134 adapter->flags &= ~FULL_INIT_DONE;
4135 }
4136
4137 /*
4138 * net_device operations
4139 */
4140 static int cxgb_open(struct net_device *dev)
4141 {
4142 int err;
4143 struct port_info *pi = netdev_priv(dev);
4144 struct adapter *adapter = pi->adapter;
4145
4146 netif_carrier_off(dev);
4147
4148 if (!(adapter->flags & FULL_INIT_DONE)) {
4149 err = cxgb_up(adapter);
4150 if (err < 0)
4151 return err;
4152 }
4153
4154 err = link_start(dev);
4155 if (!err)
4156 netif_tx_start_all_queues(dev);
4157 return err;
4158 }
4159
4160 static int cxgb_close(struct net_device *dev)
4161 {
4162 struct port_info *pi = netdev_priv(dev);
4163 struct adapter *adapter = pi->adapter;
4164
4165 netif_tx_stop_all_queues(dev);
4166 netif_carrier_off(dev);
4167 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
4168 }
4169
4170 /* Return an error number if the indicated filter isn't writable ...
4171 */
4172 static int writable_filter(struct filter_entry *f)
4173 {
4174 if (f->locked)
4175 return -EPERM;
4176 if (f->pending)
4177 return -EBUSY;
4178
4179 return 0;
4180 }
4181
4182 /* Delete the filter at the specified index (if valid). The checks for all
4183 * the common problems with doing this like the filter being locked, currently
4184 * pending in another operation, etc.
4185 */
4186 static int delete_filter(struct adapter *adapter, unsigned int fidx)
4187 {
4188 struct filter_entry *f;
4189 int ret;
4190
4191 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
4192 return -EINVAL;
4193
4194 f = &adapter->tids.ftid_tab[fidx];
4195 ret = writable_filter(f);
4196 if (ret)
4197 return ret;
4198 if (f->valid)
4199 return del_filter_wr(adapter, fidx);
4200
4201 return 0;
4202 }
4203
4204 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
4205 __be32 sip, __be16 sport, __be16 vlan,
4206 unsigned int queue, unsigned char port, unsigned char mask)
4207 {
4208 int ret;
4209 struct filter_entry *f;
4210 struct adapter *adap;
4211 int i;
4212 u8 *val;
4213
4214 adap = netdev2adap(dev);
4215
4216 /* Adjust stid to correct filter index */
4217 stid -= adap->tids.sftid_base;
4218 stid += adap->tids.nftids;
4219
4220 /* Check to make sure the filter requested is writable ...
4221 */
4222 f = &adap->tids.ftid_tab[stid];
4223 ret = writable_filter(f);
4224 if (ret)
4225 return ret;
4226
4227 /* Clear out any old resources being used by the filter before
4228 * we start constructing the new filter.
4229 */
4230 if (f->valid)
4231 clear_filter(adap, f);
4232
4233 /* Clear out filter specifications */
4234 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4235 f->fs.val.lport = cpu_to_be16(sport);
4236 f->fs.mask.lport = ~0;
4237 val = (u8 *)&sip;
4238 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
4239 for (i = 0; i < 4; i++) {
4240 f->fs.val.lip[i] = val[i];
4241 f->fs.mask.lip[i] = ~0;
4242 }
4243 if (adap->params.tp.vlan_pri_map & F_PORT) {
4244 f->fs.val.iport = port;
4245 f->fs.mask.iport = mask;
4246 }
4247 }
4248
4249 if (adap->params.tp.vlan_pri_map & F_PROTOCOL) {
4250 f->fs.val.proto = IPPROTO_TCP;
4251 f->fs.mask.proto = ~0;
4252 }
4253
4254 f->fs.dirsteer = 1;
4255 f->fs.iq = queue;
4256 /* Mark filter as locked */
4257 f->locked = 1;
4258 f->fs.rpttid = 1;
4259
4260 ret = set_filter_wr(adap, stid);
4261 if (ret) {
4262 clear_filter(adap, f);
4263 return ret;
4264 }
4265
4266 return 0;
4267 }
4268 EXPORT_SYMBOL(cxgb4_create_server_filter);
4269
4270 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4271 unsigned int queue, bool ipv6)
4272 {
4273 int ret;
4274 struct filter_entry *f;
4275 struct adapter *adap;
4276
4277 adap = netdev2adap(dev);
4278
4279 /* Adjust stid to correct filter index */
4280 stid -= adap->tids.sftid_base;
4281 stid += adap->tids.nftids;
4282
4283 f = &adap->tids.ftid_tab[stid];
4284 /* Unlock the filter */
4285 f->locked = 0;
4286
4287 ret = delete_filter(adap, stid);
4288 if (ret)
4289 return ret;
4290
4291 return 0;
4292 }
4293 EXPORT_SYMBOL(cxgb4_remove_server_filter);
4294
4295 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4296 struct rtnl_link_stats64 *ns)
4297 {
4298 struct port_stats stats;
4299 struct port_info *p = netdev_priv(dev);
4300 struct adapter *adapter = p->adapter;
4301
4302 /* Block retrieving statistics during EEH error
4303 * recovery. Otherwise, the recovery might fail
4304 * and the PCI device will be removed permanently
4305 */
4306 spin_lock(&adapter->stats_lock);
4307 if (!netif_device_present(dev)) {
4308 spin_unlock(&adapter->stats_lock);
4309 return ns;
4310 }
4311 t4_get_port_stats(adapter, p->tx_chan, &stats);
4312 spin_unlock(&adapter->stats_lock);
4313
4314 ns->tx_bytes = stats.tx_octets;
4315 ns->tx_packets = stats.tx_frames;
4316 ns->rx_bytes = stats.rx_octets;
4317 ns->rx_packets = stats.rx_frames;
4318 ns->multicast = stats.rx_mcast_frames;
4319
4320 /* detailed rx_errors */
4321 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4322 stats.rx_runt;
4323 ns->rx_over_errors = 0;
4324 ns->rx_crc_errors = stats.rx_fcs_err;
4325 ns->rx_frame_errors = stats.rx_symbol_err;
4326 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
4327 stats.rx_ovflow2 + stats.rx_ovflow3 +
4328 stats.rx_trunc0 + stats.rx_trunc1 +
4329 stats.rx_trunc2 + stats.rx_trunc3;
4330 ns->rx_missed_errors = 0;
4331
4332 /* detailed tx_errors */
4333 ns->tx_aborted_errors = 0;
4334 ns->tx_carrier_errors = 0;
4335 ns->tx_fifo_errors = 0;
4336 ns->tx_heartbeat_errors = 0;
4337 ns->tx_window_errors = 0;
4338
4339 ns->tx_errors = stats.tx_error_frames;
4340 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4341 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4342 return ns;
4343 }
4344
4345 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4346 {
4347 unsigned int mbox;
4348 int ret = 0, prtad, devad;
4349 struct port_info *pi = netdev_priv(dev);
4350 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4351
4352 switch (cmd) {
4353 case SIOCGMIIPHY:
4354 if (pi->mdio_addr < 0)
4355 return -EOPNOTSUPP;
4356 data->phy_id = pi->mdio_addr;
4357 break;
4358 case SIOCGMIIREG:
4359 case SIOCSMIIREG:
4360 if (mdio_phy_id_is_c45(data->phy_id)) {
4361 prtad = mdio_phy_id_prtad(data->phy_id);
4362 devad = mdio_phy_id_devad(data->phy_id);
4363 } else if (data->phy_id < 32) {
4364 prtad = data->phy_id;
4365 devad = 0;
4366 data->reg_num &= 0x1f;
4367 } else
4368 return -EINVAL;
4369
4370 mbox = pi->adapter->fn;
4371 if (cmd == SIOCGMIIREG)
4372 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
4373 data->reg_num, &data->val_out);
4374 else
4375 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
4376 data->reg_num, data->val_in);
4377 break;
4378 default:
4379 return -EOPNOTSUPP;
4380 }
4381 return ret;
4382 }
4383
4384 static void cxgb_set_rxmode(struct net_device *dev)
4385 {
4386 /* unfortunately we can't return errors to the stack */
4387 set_rxmode(dev, -1, false);
4388 }
4389
4390 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4391 {
4392 int ret;
4393 struct port_info *pi = netdev_priv(dev);
4394
4395 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
4396 return -EINVAL;
4397 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4398 -1, -1, -1, true);
4399 if (!ret)
4400 dev->mtu = new_mtu;
4401 return ret;
4402 }
4403
4404 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4405 {
4406 int ret;
4407 struct sockaddr *addr = p;
4408 struct port_info *pi = netdev_priv(dev);
4409
4410 if (!is_valid_ether_addr(addr->sa_data))
4411 return -EADDRNOTAVAIL;
4412
4413 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4414 pi->xact_addr_filt, addr->sa_data, true, true);
4415 if (ret < 0)
4416 return ret;
4417
4418 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4419 pi->xact_addr_filt = ret;
4420 return 0;
4421 }
4422
4423 #ifdef CONFIG_NET_POLL_CONTROLLER
4424 static void cxgb_netpoll(struct net_device *dev)
4425 {
4426 struct port_info *pi = netdev_priv(dev);
4427 struct adapter *adap = pi->adapter;
4428
4429 if (adap->flags & USING_MSIX) {
4430 int i;
4431 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4432
4433 for (i = pi->nqsets; i; i--, rx++)
4434 t4_sge_intr_msix(0, &rx->rspq);
4435 } else
4436 t4_intr_handler(adap)(0, adap);
4437 }
4438 #endif
4439
4440 static const struct net_device_ops cxgb4_netdev_ops = {
4441 .ndo_open = cxgb_open,
4442 .ndo_stop = cxgb_close,
4443 .ndo_start_xmit = t4_eth_xmit,
4444 .ndo_get_stats64 = cxgb_get_stats,
4445 .ndo_set_rx_mode = cxgb_set_rxmode,
4446 .ndo_set_mac_address = cxgb_set_mac_addr,
4447 .ndo_set_features = cxgb_set_features,
4448 .ndo_validate_addr = eth_validate_addr,
4449 .ndo_do_ioctl = cxgb_ioctl,
4450 .ndo_change_mtu = cxgb_change_mtu,
4451 #ifdef CONFIG_NET_POLL_CONTROLLER
4452 .ndo_poll_controller = cxgb_netpoll,
4453 #endif
4454 };
4455
4456 void t4_fatal_err(struct adapter *adap)
4457 {
4458 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
4459 t4_intr_disable(adap);
4460 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4461 }
4462
4463 static void setup_memwin(struct adapter *adap)
4464 {
4465 u32 bar0, mem_win0_base, mem_win1_base, mem_win2_base;
4466
4467 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
4468 if (is_t4(adap->params.chip)) {
4469 mem_win0_base = bar0 + MEMWIN0_BASE;
4470 mem_win1_base = bar0 + MEMWIN1_BASE;
4471 mem_win2_base = bar0 + MEMWIN2_BASE;
4472 } else {
4473 /* For T5, only relative offset inside the PCIe BAR is passed */
4474 mem_win0_base = MEMWIN0_BASE;
4475 mem_win1_base = MEMWIN1_BASE_T5;
4476 mem_win2_base = MEMWIN2_BASE_T5;
4477 }
4478 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
4479 mem_win0_base | BIR(0) |
4480 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
4481 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
4482 mem_win1_base | BIR(0) |
4483 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
4484 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
4485 mem_win2_base | BIR(0) |
4486 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
4487 }
4488
4489 static void setup_memwin_rdma(struct adapter *adap)
4490 {
4491 if (adap->vres.ocq.size) {
4492 unsigned int start, sz_kb;
4493
4494 start = pci_resource_start(adap->pdev, 2) +
4495 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4496 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4497 t4_write_reg(adap,
4498 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
4499 start | BIR(1) | WINDOW(ilog2(sz_kb)));
4500 t4_write_reg(adap,
4501 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
4502 adap->vres.ocq.start);
4503 t4_read_reg(adap,
4504 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
4505 }
4506 }
4507
4508 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4509 {
4510 u32 v;
4511 int ret;
4512
4513 /* get device capabilities */
4514 memset(c, 0, sizeof(*c));
4515 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4516 FW_CMD_REQUEST | FW_CMD_READ);
4517 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
4518 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
4519 if (ret < 0)
4520 return ret;
4521
4522 /* select capabilities we'll be using */
4523 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4524 if (!vf_acls)
4525 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4526 else
4527 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4528 } else if (vf_acls) {
4529 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4530 return ret;
4531 }
4532 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4533 FW_CMD_REQUEST | FW_CMD_WRITE);
4534 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
4535 if (ret < 0)
4536 return ret;
4537
4538 ret = t4_config_glbl_rss(adap, adap->fn,
4539 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4540 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4541 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
4542 if (ret < 0)
4543 return ret;
4544
4545 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4546 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
4547 if (ret < 0)
4548 return ret;
4549
4550 t4_sge_init(adap);
4551
4552 /* tweak some settings */
4553 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
4554 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
4555 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
4556 v = t4_read_reg(adap, TP_PIO_DATA);
4557 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
4558
4559 /* first 4 Tx modulation queues point to consecutive Tx channels */
4560 adap->params.tp.tx_modq_map = 0xE4;
4561 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP,
4562 V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map));
4563
4564 /* associate each Tx modulation queue with consecutive Tx channels */
4565 v = 0x84218421;
4566 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4567 &v, 1, A_TP_TX_SCHED_HDR);
4568 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4569 &v, 1, A_TP_TX_SCHED_FIFO);
4570 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4571 &v, 1, A_TP_TX_SCHED_PCMD);
4572
4573 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4574 if (is_offload(adap)) {
4575 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0,
4576 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4577 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4578 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4579 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4580 t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT,
4581 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4582 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4583 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4584 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4585 }
4586
4587 /* get basic stuff going */
4588 return t4_early_init(adap, adap->fn);
4589 }
4590
4591 /*
4592 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4593 */
4594 #define MAX_ATIDS 8192U
4595
4596 /*
4597 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4598 *
4599 * If the firmware we're dealing with has Configuration File support, then
4600 * we use that to perform all configuration
4601 */
4602
4603 /*
4604 * Tweak configuration based on module parameters, etc. Most of these have
4605 * defaults assigned to them by Firmware Configuration Files (if we're using
4606 * them) but need to be explicitly set if we're using hard-coded
4607 * initialization. But even in the case of using Firmware Configuration
4608 * Files, we'd like to expose the ability to change these via module
4609 * parameters so these are essentially common tweaks/settings for
4610 * Configuration Files and hard-coded initialization ...
4611 */
4612 static int adap_init0_tweaks(struct adapter *adapter)
4613 {
4614 /*
4615 * Fix up various Host-Dependent Parameters like Page Size, Cache
4616 * Line Size, etc. The firmware default is for a 4KB Page Size and
4617 * 64B Cache Line Size ...
4618 */
4619 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4620
4621 /*
4622 * Process module parameters which affect early initialization.
4623 */
4624 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4625 dev_err(&adapter->pdev->dev,
4626 "Ignoring illegal rx_dma_offset=%d, using 2\n",
4627 rx_dma_offset);
4628 rx_dma_offset = 2;
4629 }
4630 t4_set_reg_field(adapter, SGE_CONTROL,
4631 PKTSHIFT_MASK,
4632 PKTSHIFT(rx_dma_offset));
4633
4634 /*
4635 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4636 * adds the pseudo header itself.
4637 */
4638 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
4639 CSUM_HAS_PSEUDO_HDR, 0);
4640
4641 return 0;
4642 }
4643
4644 /*
4645 * Attempt to initialize the adapter via a Firmware Configuration File.
4646 */
4647 static int adap_init0_config(struct adapter *adapter, int reset)
4648 {
4649 struct fw_caps_config_cmd caps_cmd;
4650 const struct firmware *cf;
4651 unsigned long mtype = 0, maddr = 0;
4652 u32 finiver, finicsum, cfcsum;
4653 int ret;
4654 int config_issued = 0;
4655 char *fw_config_file, fw_config_file_path[256];
4656 char *config_name = NULL;
4657
4658 /*
4659 * Reset device if necessary.
4660 */
4661 if (reset) {
4662 ret = t4_fw_reset(adapter, adapter->mbox,
4663 PIORSTMODE | PIORST);
4664 if (ret < 0)
4665 goto bye;
4666 }
4667
4668 /*
4669 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4670 * then use that. Otherwise, use the configuration file stored
4671 * in the adapter flash ...
4672 */
4673 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
4674 case CHELSIO_T4:
4675 fw_config_file = FW4_CFNAME;
4676 break;
4677 case CHELSIO_T5:
4678 fw_config_file = FW5_CFNAME;
4679 break;
4680 default:
4681 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4682 adapter->pdev->device);
4683 ret = -EINVAL;
4684 goto bye;
4685 }
4686
4687 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
4688 if (ret < 0) {
4689 config_name = "On FLASH";
4690 mtype = FW_MEMTYPE_CF_FLASH;
4691 maddr = t4_flash_cfg_addr(adapter);
4692 } else {
4693 u32 params[7], val[7];
4694
4695 sprintf(fw_config_file_path,
4696 "/lib/firmware/%s", fw_config_file);
4697 config_name = fw_config_file_path;
4698
4699 if (cf->size >= FLASH_CFG_MAX_SIZE)
4700 ret = -ENOMEM;
4701 else {
4702 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4703 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
4704 ret = t4_query_params(adapter, adapter->mbox,
4705 adapter->fn, 0, 1, params, val);
4706 if (ret == 0) {
4707 /*
4708 * For t4_memory_write() below addresses and
4709 * sizes have to be in terms of multiples of 4
4710 * bytes. So, if the Configuration File isn't
4711 * a multiple of 4 bytes in length we'll have
4712 * to write that out separately since we can't
4713 * guarantee that the bytes following the
4714 * residual byte in the buffer returned by
4715 * request_firmware() are zeroed out ...
4716 */
4717 size_t resid = cf->size & 0x3;
4718 size_t size = cf->size & ~0x3;
4719 __be32 *data = (__be32 *)cf->data;
4720
4721 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);
4722 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;
4723
4724 ret = t4_memory_write(adapter, mtype, maddr,
4725 size, data);
4726 if (ret == 0 && resid != 0) {
4727 union {
4728 __be32 word;
4729 char buf[4];
4730 } last;
4731 int i;
4732
4733 last.word = data[size >> 2];
4734 for (i = resid; i < 4; i++)
4735 last.buf[i] = 0;
4736 ret = t4_memory_write(adapter, mtype,
4737 maddr + size,
4738 4, &last.word);
4739 }
4740 }
4741 }
4742
4743 release_firmware(cf);
4744 if (ret)
4745 goto bye;
4746 }
4747
4748 /*
4749 * Issue a Capability Configuration command to the firmware to get it
4750 * to parse the Configuration File. We don't use t4_fw_config_file()
4751 * because we want the ability to modify various features after we've
4752 * processed the configuration file ...
4753 */
4754 memset(&caps_cmd, 0, sizeof(caps_cmd));
4755 caps_cmd.op_to_write =
4756 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4757 FW_CMD_REQUEST |
4758 FW_CMD_READ);
4759 caps_cmd.cfvalid_to_len16 =
4760 htonl(FW_CAPS_CONFIG_CMD_CFVALID |
4761 FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
4762 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
4763 FW_LEN16(caps_cmd));
4764 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4765 &caps_cmd);
4766
4767 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4768 * Configuration File in FLASH), our last gasp effort is to use the
4769 * Firmware Configuration File which is embedded in the firmware. A
4770 * very few early versions of the firmware didn't have one embedded
4771 * but we can ignore those.
4772 */
4773 if (ret == -ENOENT) {
4774 memset(&caps_cmd, 0, sizeof(caps_cmd));
4775 caps_cmd.op_to_write =
4776 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4777 FW_CMD_REQUEST |
4778 FW_CMD_READ);
4779 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4780 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4781 sizeof(caps_cmd), &caps_cmd);
4782 config_name = "Firmware Default";
4783 }
4784
4785 config_issued = 1;
4786 if (ret < 0)
4787 goto bye;
4788
4789 finiver = ntohl(caps_cmd.finiver);
4790 finicsum = ntohl(caps_cmd.finicsum);
4791 cfcsum = ntohl(caps_cmd.cfcsum);
4792 if (finicsum != cfcsum)
4793 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4794 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4795 finicsum, cfcsum);
4796
4797 /*
4798 * And now tell the firmware to use the configuration we just loaded.
4799 */
4800 caps_cmd.op_to_write =
4801 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4802 FW_CMD_REQUEST |
4803 FW_CMD_WRITE);
4804 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4805 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4806 NULL);
4807 if (ret < 0)
4808 goto bye;
4809
4810 /*
4811 * Tweak configuration based on system architecture, module
4812 * parameters, etc.
4813 */
4814 ret = adap_init0_tweaks(adapter);
4815 if (ret < 0)
4816 goto bye;
4817
4818 /*
4819 * And finally tell the firmware to initialize itself using the
4820 * parameters from the Configuration File.
4821 */
4822 ret = t4_fw_initialize(adapter, adapter->mbox);
4823 if (ret < 0)
4824 goto bye;
4825
4826 /*
4827 * Return successfully and note that we're operating with parameters
4828 * not supplied by the driver, rather than from hard-wired
4829 * initialization constants burried in the driver.
4830 */
4831 adapter->flags |= USING_SOFT_PARAMS;
4832 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
4833 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
4834 config_name, finiver, cfcsum);
4835 return 0;
4836
4837 /*
4838 * Something bad happened. Return the error ... (If the "error"
4839 * is that there's no Configuration File on the adapter we don't
4840 * want to issue a warning since this is fairly common.)
4841 */
4842 bye:
4843 if (config_issued && ret != -ENOENT)
4844 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
4845 config_name, -ret);
4846 return ret;
4847 }
4848
4849 /*
4850 * Attempt to initialize the adapter via hard-coded, driver supplied
4851 * parameters ...
4852 */
4853 static int adap_init0_no_config(struct adapter *adapter, int reset)
4854 {
4855 struct sge *s = &adapter->sge;
4856 struct fw_caps_config_cmd caps_cmd;
4857 u32 v;
4858 int i, ret;
4859
4860 /*
4861 * Reset device if necessary
4862 */
4863 if (reset) {
4864 ret = t4_fw_reset(adapter, adapter->mbox,
4865 PIORSTMODE | PIORST);
4866 if (ret < 0)
4867 goto bye;
4868 }
4869
4870 /*
4871 * Get device capabilities and select which we'll be using.
4872 */
4873 memset(&caps_cmd, 0, sizeof(caps_cmd));
4874 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4875 FW_CMD_REQUEST | FW_CMD_READ);
4876 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4877 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4878 &caps_cmd);
4879 if (ret < 0)
4880 goto bye;
4881
4882 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4883 if (!vf_acls)
4884 caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4885 else
4886 caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4887 } else if (vf_acls) {
4888 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
4889 goto bye;
4890 }
4891 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4892 FW_CMD_REQUEST | FW_CMD_WRITE);
4893 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4894 NULL);
4895 if (ret < 0)
4896 goto bye;
4897
4898 /*
4899 * Tweak configuration based on system architecture, module
4900 * parameters, etc.
4901 */
4902 ret = adap_init0_tweaks(adapter);
4903 if (ret < 0)
4904 goto bye;
4905
4906 /*
4907 * Select RSS Global Mode we want to use. We use "Basic Virtual"
4908 * mode which maps each Virtual Interface to its own section of
4909 * the RSS Table and we turn on all map and hash enables ...
4910 */
4911 adapter->flags |= RSS_TNLALLLOOKUP;
4912 ret = t4_config_glbl_rss(adapter, adapter->mbox,
4913 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4914 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4915 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |
4916 ((adapter->flags & RSS_TNLALLLOOKUP) ?
4917 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP : 0));
4918 if (ret < 0)
4919 goto bye;
4920
4921 /*
4922 * Set up our own fundamental resource provisioning ...
4923 */
4924 ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
4925 PFRES_NEQ, PFRES_NETHCTRL,
4926 PFRES_NIQFLINT, PFRES_NIQ,
4927 PFRES_TC, PFRES_NVI,
4928 FW_PFVF_CMD_CMASK_MASK,
4929 pfvfres_pmask(adapter, adapter->fn, 0),
4930 PFRES_NEXACTF,
4931 PFRES_R_CAPS, PFRES_WX_CAPS);
4932 if (ret < 0)
4933 goto bye;
4934
4935 /*
4936 * Perform low level SGE initialization. We need to do this before we
4937 * send the firmware the INITIALIZE command because that will cause
4938 * any other PF Drivers which are waiting for the Master
4939 * Initialization to proceed forward.
4940 */
4941 for (i = 0; i < SGE_NTIMERS - 1; i++)
4942 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
4943 s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
4944 s->counter_val[0] = 1;
4945 for (i = 1; i < SGE_NCOUNTERS; i++)
4946 s->counter_val[i] = min(intr_cnt[i - 1],
4947 THRESHOLD_0_GET(THRESHOLD_0_MASK));
4948 t4_sge_init(adapter);
4949
4950 #ifdef CONFIG_PCI_IOV
4951 /*
4952 * Provision resource limits for Virtual Functions. We currently
4953 * grant them all the same static resource limits except for the Port
4954 * Access Rights Mask which we're assigning based on the PF. All of
4955 * the static provisioning stuff for both the PF and VF really needs
4956 * to be managed in a persistent manner for each device which the
4957 * firmware controls.
4958 */
4959 {
4960 int pf, vf;
4961
4962 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
4963 if (num_vf[pf] <= 0)
4964 continue;
4965
4966 /* VF numbering starts at 1! */
4967 for (vf = 1; vf <= num_vf[pf]; vf++) {
4968 ret = t4_cfg_pfvf(adapter, adapter->mbox,
4969 pf, vf,
4970 VFRES_NEQ, VFRES_NETHCTRL,
4971 VFRES_NIQFLINT, VFRES_NIQ,
4972 VFRES_TC, VFRES_NVI,
4973 FW_PFVF_CMD_CMASK_MASK,
4974 pfvfres_pmask(
4975 adapter, pf, vf),
4976 VFRES_NEXACTF,
4977 VFRES_R_CAPS, VFRES_WX_CAPS);
4978 if (ret < 0)
4979 dev_warn(adapter->pdev_dev,
4980 "failed to "\
4981 "provision pf/vf=%d/%d; "
4982 "err=%d\n", pf, vf, ret);
4983 }
4984 }
4985 }
4986 #endif
4987
4988 /*
4989 * Set up the default filter mode. Later we'll want to implement this
4990 * via a firmware command, etc. ... This needs to be done before the
4991 * firmare initialization command ... If the selected set of fields
4992 * isn't equal to the default value, we'll need to make sure that the
4993 * field selections will fit in the 36-bit budget.
4994 */
4995 if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
4996 int j, bits = 0;
4997
4998 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)
4999 switch (tp_vlan_pri_map & (1 << j)) {
5000 case 0:
5001 /* compressed filter field not enabled */
5002 break;
5003 case FCOE_MASK:
5004 bits += 1;
5005 break;
5006 case PORT_MASK:
5007 bits += 3;
5008 break;
5009 case VNIC_ID_MASK:
5010 bits += 17;
5011 break;
5012 case VLAN_MASK:
5013 bits += 17;
5014 break;
5015 case TOS_MASK:
5016 bits += 8;
5017 break;
5018 case PROTOCOL_MASK:
5019 bits += 8;
5020 break;
5021 case ETHERTYPE_MASK:
5022 bits += 16;
5023 break;
5024 case MACMATCH_MASK:
5025 bits += 9;
5026 break;
5027 case MPSHITTYPE_MASK:
5028 bits += 3;
5029 break;
5030 case FRAGMENTATION_MASK:
5031 bits += 1;
5032 break;
5033 }
5034
5035 if (bits > 36) {
5036 dev_err(adapter->pdev_dev,
5037 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
5038 " using %#x\n", tp_vlan_pri_map, bits,
5039 TP_VLAN_PRI_MAP_DEFAULT);
5040 tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
5041 }
5042 }
5043 v = tp_vlan_pri_map;
5044 t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,
5045 &v, 1, TP_VLAN_PRI_MAP);
5046
5047 /*
5048 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
5049 * to support any of the compressed filter fields above. Newer
5050 * versions of the firmware do this automatically but it doesn't hurt
5051 * to set it here. Meanwhile, we do _not_ need to set Lookup Every
5052 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
5053 * since the firmware automatically turns this on and off when we have
5054 * a non-zero number of filters active (since it does have a
5055 * performance impact).
5056 */
5057 if (tp_vlan_pri_map)
5058 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,
5059 FIVETUPLELOOKUP_MASK,
5060 FIVETUPLELOOKUP_MASK);
5061
5062 /*
5063 * Tweak some settings.
5064 */
5065 t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |
5066 RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
5067 PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
5068 KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
5069
5070 /*
5071 * Get basic stuff going by issuing the Firmware Initialize command.
5072 * Note that this _must_ be after all PFVF commands ...
5073 */
5074 ret = t4_fw_initialize(adapter, adapter->mbox);
5075 if (ret < 0)
5076 goto bye;
5077
5078 /*
5079 * Return successfully!
5080 */
5081 dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
5082 "driver parameters\n");
5083 return 0;
5084
5085 /*
5086 * Something bad happened. Return the error ...
5087 */
5088 bye:
5089 return ret;
5090 }
5091
5092 static struct fw_info fw_info_array[] = {
5093 {
5094 .chip = CHELSIO_T4,
5095 .fs_name = FW4_CFNAME,
5096 .fw_mod_name = FW4_FNAME,
5097 .fw_hdr = {
5098 .chip = FW_HDR_CHIP_T4,
5099 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5100 .intfver_nic = FW_INTFVER(T4, NIC),
5101 .intfver_vnic = FW_INTFVER(T4, VNIC),
5102 .intfver_ri = FW_INTFVER(T4, RI),
5103 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5104 .intfver_fcoe = FW_INTFVER(T4, FCOE),
5105 },
5106 }, {
5107 .chip = CHELSIO_T5,
5108 .fs_name = FW5_CFNAME,
5109 .fw_mod_name = FW5_FNAME,
5110 .fw_hdr = {
5111 .chip = FW_HDR_CHIP_T5,
5112 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5113 .intfver_nic = FW_INTFVER(T5, NIC),
5114 .intfver_vnic = FW_INTFVER(T5, VNIC),
5115 .intfver_ri = FW_INTFVER(T5, RI),
5116 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5117 .intfver_fcoe = FW_INTFVER(T5, FCOE),
5118 },
5119 }
5120 };
5121
5122 static struct fw_info *find_fw_info(int chip)
5123 {
5124 int i;
5125
5126 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5127 if (fw_info_array[i].chip == chip)
5128 return &fw_info_array[i];
5129 }
5130 return NULL;
5131 }
5132
5133 /*
5134 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5135 */
5136 static int adap_init0(struct adapter *adap)
5137 {
5138 int ret;
5139 u32 v, port_vec;
5140 enum dev_state state;
5141 u32 params[7], val[7];
5142 struct fw_caps_config_cmd caps_cmd;
5143 int reset = 1;
5144
5145 /*
5146 * Contact FW, advertising Master capability (and potentially forcing
5147 * ourselves as the Master PF if our module parameter force_init is
5148 * set).
5149 */
5150 ret = t4_fw_hello(adap, adap->mbox, adap->fn,
5151 force_init ? MASTER_MUST : MASTER_MAY,
5152 &state);
5153 if (ret < 0) {
5154 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5155 ret);
5156 return ret;
5157 }
5158 if (ret == adap->mbox)
5159 adap->flags |= MASTER_PF;
5160 if (force_init && state == DEV_STATE_INIT)
5161 state = DEV_STATE_UNINIT;
5162
5163 /*
5164 * If we're the Master PF Driver and the device is uninitialized,
5165 * then let's consider upgrading the firmware ... (We always want
5166 * to check the firmware version number in order to A. get it for
5167 * later reporting and B. to warn if the currently loaded firmware
5168 * is excessively mismatched relative to the driver.)
5169 */
5170 t4_get_fw_version(adap, &adap->params.fw_vers);
5171 t4_get_tp_version(adap, &adap->params.tp_vers);
5172 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
5173 struct fw_info *fw_info;
5174 struct fw_hdr *card_fw;
5175 const struct firmware *fw;
5176 const u8 *fw_data = NULL;
5177 unsigned int fw_size = 0;
5178
5179 /* This is the firmware whose headers the driver was compiled
5180 * against
5181 */
5182 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5183 if (fw_info == NULL) {
5184 dev_err(adap->pdev_dev,
5185 "unable to get firmware info for chip %d.\n",
5186 CHELSIO_CHIP_VERSION(adap->params.chip));
5187 return -EINVAL;
5188 }
5189
5190 /* allocate memory to read the header of the firmware on the
5191 * card
5192 */
5193 card_fw = t4_alloc_mem(sizeof(*card_fw));
5194
5195 /* Get FW from from /lib/firmware/ */
5196 ret = request_firmware(&fw, fw_info->fw_mod_name,
5197 adap->pdev_dev);
5198 if (ret < 0) {
5199 dev_err(adap->pdev_dev,
5200 "unable to load firmware image %s, error %d\n",
5201 fw_info->fw_mod_name, ret);
5202 } else {
5203 fw_data = fw->data;
5204 fw_size = fw->size;
5205 }
5206
5207 /* upgrade FW logic */
5208 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5209 state, &reset);
5210
5211 /* Cleaning up */
5212 if (fw != NULL)
5213 release_firmware(fw);
5214 t4_free_mem(card_fw);
5215
5216 if (ret < 0)
5217 goto bye;
5218 }
5219
5220 /*
5221 * Grab VPD parameters. This should be done after we establish a
5222 * connection to the firmware since some of the VPD parameters
5223 * (notably the Core Clock frequency) are retrieved via requests to
5224 * the firmware. On the other hand, we need these fairly early on
5225 * so we do this right after getting ahold of the firmware.
5226 */
5227 ret = get_vpd_params(adap, &adap->params.vpd);
5228 if (ret < 0)
5229 goto bye;
5230
5231 /*
5232 * Find out what ports are available to us. Note that we need to do
5233 * this before calling adap_init0_no_config() since it needs nports
5234 * and portvec ...
5235 */
5236 v =
5237 FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5238 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
5239 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
5240 if (ret < 0)
5241 goto bye;
5242
5243 adap->params.nports = hweight32(port_vec);
5244 adap->params.portvec = port_vec;
5245
5246 /*
5247 * If the firmware is initialized already (and we're not forcing a
5248 * master initialization), note that we're living with existing
5249 * adapter parameters. Otherwise, it's time to try initializing the
5250 * adapter ...
5251 */
5252 if (state == DEV_STATE_INIT) {
5253 dev_info(adap->pdev_dev, "Coming up as %s: "\
5254 "Adapter already initialized\n",
5255 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5256 adap->flags |= USING_SOFT_PARAMS;
5257 } else {
5258 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5259 "Initializing adapter\n");
5260
5261 /*
5262 * If the firmware doesn't support Configuration
5263 * Files warn user and exit,
5264 */
5265 if (ret < 0)
5266 dev_warn(adap->pdev_dev, "Firmware doesn't support "
5267 "configuration file.\n");
5268 if (force_old_init)
5269 ret = adap_init0_no_config(adap, reset);
5270 else {
5271 /*
5272 * Find out whether we're dealing with a version of
5273 * the firmware which has configuration file support.
5274 */
5275 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5276 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
5277 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5278 params, val);
5279
5280 /*
5281 * If the firmware doesn't support Configuration
5282 * Files, use the old Driver-based, hard-wired
5283 * initialization. Otherwise, try using the
5284 * Configuration File support and fall back to the
5285 * Driver-based initialization if there's no
5286 * Configuration File found.
5287 */
5288 if (ret < 0)
5289 ret = adap_init0_no_config(adap, reset);
5290 else {
5291 /*
5292 * The firmware provides us with a memory
5293 * buffer where we can load a Configuration
5294 * File from the host if we want to override
5295 * the Configuration File in flash.
5296 */
5297
5298 ret = adap_init0_config(adap, reset);
5299 if (ret == -ENOENT) {
5300 dev_info(adap->pdev_dev,
5301 "No Configuration File present "
5302 "on adapter. Using hard-wired "
5303 "configuration parameters.\n");
5304 ret = adap_init0_no_config(adap, reset);
5305 }
5306 }
5307 }
5308 if (ret < 0) {
5309 dev_err(adap->pdev_dev,
5310 "could not initialize adapter, error %d\n",
5311 -ret);
5312 goto bye;
5313 }
5314 }
5315
5316 /*
5317 * If we're living with non-hard-coded parameters (either from a
5318 * Firmware Configuration File or values programmed by a different PF
5319 * Driver), give the SGE code a chance to pull in anything that it
5320 * needs ... Note that this must be called after we retrieve our VPD
5321 * parameters in order to know how to convert core ticks to seconds.
5322 */
5323 if (adap->flags & USING_SOFT_PARAMS) {
5324 ret = t4_sge_init(adap);
5325 if (ret < 0)
5326 goto bye;
5327 }
5328
5329 if (is_bypass_device(adap->pdev->device))
5330 adap->params.bypass = 1;
5331
5332 /*
5333 * Grab some of our basic fundamental operating parameters.
5334 */
5335 #define FW_PARAM_DEV(param) \
5336 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
5337 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
5338
5339 #define FW_PARAM_PFVF(param) \
5340 FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
5341 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)| \
5342 FW_PARAMS_PARAM_Y(0) | \
5343 FW_PARAMS_PARAM_Z(0)
5344
5345 params[0] = FW_PARAM_PFVF(EQ_START);
5346 params[1] = FW_PARAM_PFVF(L2T_START);
5347 params[2] = FW_PARAM_PFVF(L2T_END);
5348 params[3] = FW_PARAM_PFVF(FILTER_START);
5349 params[4] = FW_PARAM_PFVF(FILTER_END);
5350 params[5] = FW_PARAM_PFVF(IQFLINT_START);
5351 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
5352 if (ret < 0)
5353 goto bye;
5354 adap->sge.egr_start = val[0];
5355 adap->l2t_start = val[1];
5356 adap->l2t_end = val[2];
5357 adap->tids.ftid_base = val[3];
5358 adap->tids.nftids = val[4] - val[3] + 1;
5359 adap->sge.ingr_start = val[5];
5360
5361 /* query params related to active filter region */
5362 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5363 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5364 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5365 /* If Active filter size is set we enable establishing
5366 * offload connection through firmware work request
5367 */
5368 if ((val[0] != val[1]) && (ret >= 0)) {
5369 adap->flags |= FW_OFLD_CONN;
5370 adap->tids.aftid_base = val[0];
5371 adap->tids.aftid_end = val[1];
5372 }
5373
5374 /* If we're running on newer firmware, let it know that we're
5375 * prepared to deal with encapsulated CPL messages. Older
5376 * firmware won't understand this and we'll just get
5377 * unencapsulated messages ...
5378 */
5379 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5380 val[0] = 1;
5381 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5382
5383 /*
5384 * Get device capabilities so we can determine what resources we need
5385 * to manage.
5386 */
5387 memset(&caps_cmd, 0, sizeof(caps_cmd));
5388 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
5389 FW_CMD_REQUEST | FW_CMD_READ);
5390 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5391 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5392 &caps_cmd);
5393 if (ret < 0)
5394 goto bye;
5395
5396 if (caps_cmd.ofldcaps) {
5397 /* query offload-related parameters */
5398 params[0] = FW_PARAM_DEV(NTID);
5399 params[1] = FW_PARAM_PFVF(SERVER_START);
5400 params[2] = FW_PARAM_PFVF(SERVER_END);
5401 params[3] = FW_PARAM_PFVF(TDDP_START);
5402 params[4] = FW_PARAM_PFVF(TDDP_END);
5403 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
5404 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5405 params, val);
5406 if (ret < 0)
5407 goto bye;
5408 adap->tids.ntids = val[0];
5409 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5410 adap->tids.stid_base = val[1];
5411 adap->tids.nstids = val[2] - val[1] + 1;
5412 /*
5413 * Setup server filter region. Divide the availble filter
5414 * region into two parts. Regular filters get 1/3rd and server
5415 * filters get 2/3rd part. This is only enabled if workarond
5416 * path is enabled.
5417 * 1. For regular filters.
5418 * 2. Server filter: This are special filters which are used
5419 * to redirect SYN packets to offload queue.
5420 */
5421 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5422 adap->tids.sftid_base = adap->tids.ftid_base +
5423 DIV_ROUND_UP(adap->tids.nftids, 3);
5424 adap->tids.nsftids = adap->tids.nftids -
5425 DIV_ROUND_UP(adap->tids.nftids, 3);
5426 adap->tids.nftids = adap->tids.sftid_base -
5427 adap->tids.ftid_base;
5428 }
5429 adap->vres.ddp.start = val[3];
5430 adap->vres.ddp.size = val[4] - val[3] + 1;
5431 adap->params.ofldq_wr_cred = val[5];
5432
5433 adap->params.offload = 1;
5434 }
5435 if (caps_cmd.rdmacaps) {
5436 params[0] = FW_PARAM_PFVF(STAG_START);
5437 params[1] = FW_PARAM_PFVF(STAG_END);
5438 params[2] = FW_PARAM_PFVF(RQ_START);
5439 params[3] = FW_PARAM_PFVF(RQ_END);
5440 params[4] = FW_PARAM_PFVF(PBL_START);
5441 params[5] = FW_PARAM_PFVF(PBL_END);
5442 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5443 params, val);
5444 if (ret < 0)
5445 goto bye;
5446 adap->vres.stag.start = val[0];
5447 adap->vres.stag.size = val[1] - val[0] + 1;
5448 adap->vres.rq.start = val[2];
5449 adap->vres.rq.size = val[3] - val[2] + 1;
5450 adap->vres.pbl.start = val[4];
5451 adap->vres.pbl.size = val[5] - val[4] + 1;
5452
5453 params[0] = FW_PARAM_PFVF(SQRQ_START);
5454 params[1] = FW_PARAM_PFVF(SQRQ_END);
5455 params[2] = FW_PARAM_PFVF(CQ_START);
5456 params[3] = FW_PARAM_PFVF(CQ_END);
5457 params[4] = FW_PARAM_PFVF(OCQ_START);
5458 params[5] = FW_PARAM_PFVF(OCQ_END);
5459 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
5460 if (ret < 0)
5461 goto bye;
5462 adap->vres.qp.start = val[0];
5463 adap->vres.qp.size = val[1] - val[0] + 1;
5464 adap->vres.cq.start = val[2];
5465 adap->vres.cq.size = val[3] - val[2] + 1;
5466 adap->vres.ocq.start = val[4];
5467 adap->vres.ocq.size = val[5] - val[4] + 1;
5468 }
5469 if (caps_cmd.iscsicaps) {
5470 params[0] = FW_PARAM_PFVF(ISCSI_START);
5471 params[1] = FW_PARAM_PFVF(ISCSI_END);
5472 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5473 params, val);
5474 if (ret < 0)
5475 goto bye;
5476 adap->vres.iscsi.start = val[0];
5477 adap->vres.iscsi.size = val[1] - val[0] + 1;
5478 }
5479 #undef FW_PARAM_PFVF
5480 #undef FW_PARAM_DEV
5481
5482 /*
5483 * These are finalized by FW initialization, load their values now.
5484 */
5485 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5486 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5487 adap->params.b_wnd);
5488
5489 t4_init_tp_params(adap);
5490 adap->flags |= FW_OK;
5491 return 0;
5492
5493 /*
5494 * Something bad happened. If a command timed out or failed with EIO
5495 * FW does not operate within its spec or something catastrophic
5496 * happened to HW/FW, stop issuing commands.
5497 */
5498 bye:
5499 if (ret != -ETIMEDOUT && ret != -EIO)
5500 t4_fw_bye(adap, adap->mbox);
5501 return ret;
5502 }
5503
5504 /* EEH callbacks */
5505
5506 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5507 pci_channel_state_t state)
5508 {
5509 int i;
5510 struct adapter *adap = pci_get_drvdata(pdev);
5511
5512 if (!adap)
5513 goto out;
5514
5515 rtnl_lock();
5516 adap->flags &= ~FW_OK;
5517 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
5518 spin_lock(&adap->stats_lock);
5519 for_each_port(adap, i) {
5520 struct net_device *dev = adap->port[i];
5521
5522 netif_device_detach(dev);
5523 netif_carrier_off(dev);
5524 }
5525 spin_unlock(&adap->stats_lock);
5526 if (adap->flags & FULL_INIT_DONE)
5527 cxgb_down(adap);
5528 rtnl_unlock();
5529 if ((adap->flags & DEV_ENABLED)) {
5530 pci_disable_device(pdev);
5531 adap->flags &= ~DEV_ENABLED;
5532 }
5533 out: return state == pci_channel_io_perm_failure ?
5534 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5535 }
5536
5537 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5538 {
5539 int i, ret;
5540 struct fw_caps_config_cmd c;
5541 struct adapter *adap = pci_get_drvdata(pdev);
5542
5543 if (!adap) {
5544 pci_restore_state(pdev);
5545 pci_save_state(pdev);
5546 return PCI_ERS_RESULT_RECOVERED;
5547 }
5548
5549 if (!(adap->flags & DEV_ENABLED)) {
5550 if (pci_enable_device(pdev)) {
5551 dev_err(&pdev->dev, "Cannot reenable PCI "
5552 "device after reset\n");
5553 return PCI_ERS_RESULT_DISCONNECT;
5554 }
5555 adap->flags |= DEV_ENABLED;
5556 }
5557
5558 pci_set_master(pdev);
5559 pci_restore_state(pdev);
5560 pci_save_state(pdev);
5561 pci_cleanup_aer_uncorrect_error_status(pdev);
5562
5563 if (t4_wait_dev_ready(adap) < 0)
5564 return PCI_ERS_RESULT_DISCONNECT;
5565 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
5566 return PCI_ERS_RESULT_DISCONNECT;
5567 adap->flags |= FW_OK;
5568 if (adap_init1(adap, &c))
5569 return PCI_ERS_RESULT_DISCONNECT;
5570
5571 for_each_port(adap, i) {
5572 struct port_info *p = adap2pinfo(adap, i);
5573
5574 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5575 NULL, NULL);
5576 if (ret < 0)
5577 return PCI_ERS_RESULT_DISCONNECT;
5578 p->viid = ret;
5579 p->xact_addr_filt = -1;
5580 }
5581
5582 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5583 adap->params.b_wnd);
5584 setup_memwin(adap);
5585 if (cxgb_up(adap))
5586 return PCI_ERS_RESULT_DISCONNECT;
5587 return PCI_ERS_RESULT_RECOVERED;
5588 }
5589
5590 static void eeh_resume(struct pci_dev *pdev)
5591 {
5592 int i;
5593 struct adapter *adap = pci_get_drvdata(pdev);
5594
5595 if (!adap)
5596 return;
5597
5598 rtnl_lock();
5599 for_each_port(adap, i) {
5600 struct net_device *dev = adap->port[i];
5601
5602 if (netif_running(dev)) {
5603 link_start(dev);
5604 cxgb_set_rxmode(dev);
5605 }
5606 netif_device_attach(dev);
5607 }
5608 rtnl_unlock();
5609 }
5610
5611 static const struct pci_error_handlers cxgb4_eeh = {
5612 .error_detected = eeh_err_detected,
5613 .slot_reset = eeh_slot_reset,
5614 .resume = eeh_resume,
5615 };
5616
5617 static inline bool is_10g_port(const struct link_config *lc)
5618 {
5619 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
5620 }
5621
5622 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
5623 unsigned int size, unsigned int iqe_size)
5624 {
5625 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
5626 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
5627 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
5628 q->iqe_len = iqe_size;
5629 q->size = size;
5630 }
5631
5632 /*
5633 * Perform default configuration of DMA queues depending on the number and type
5634 * of ports we found and the number of available CPUs. Most settings can be
5635 * modified by the admin prior to actual use.
5636 */
5637 static void cfg_queues(struct adapter *adap)
5638 {
5639 struct sge *s = &adap->sge;
5640 int i, q10g = 0, n10g = 0, qidx = 0;
5641
5642 for_each_port(adap, i)
5643 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
5644
5645 /*
5646 * We default to 1 queue per non-10G port and up to # of cores queues
5647 * per 10G port.
5648 */
5649 if (n10g)
5650 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5651 if (q10g > netif_get_num_default_rss_queues())
5652 q10g = netif_get_num_default_rss_queues();
5653
5654 for_each_port(adap, i) {
5655 struct port_info *pi = adap2pinfo(adap, i);
5656
5657 pi->first_qset = qidx;
5658 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
5659 qidx += pi->nqsets;
5660 }
5661
5662 s->ethqsets = qidx;
5663 s->max_ethqsets = qidx; /* MSI-X may lower it later */
5664
5665 if (is_offload(adap)) {
5666 /*
5667 * For offload we use 1 queue/channel if all ports are up to 1G,
5668 * otherwise we divide all available queues amongst the channels
5669 * capped by the number of available cores.
5670 */
5671 if (n10g) {
5672 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5673 num_online_cpus());
5674 s->ofldqsets = roundup(i, adap->params.nports);
5675 } else
5676 s->ofldqsets = adap->params.nports;
5677 /* For RDMA one Rx queue per channel suffices */
5678 s->rdmaqs = adap->params.nports;
5679 }
5680
5681 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5682 struct sge_eth_rxq *r = &s->ethrxq[i];
5683
5684 init_rspq(&r->rspq, 0, 0, 1024, 64);
5685 r->fl.size = 72;
5686 }
5687
5688 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5689 s->ethtxq[i].q.size = 1024;
5690
5691 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5692 s->ctrlq[i].q.size = 512;
5693
5694 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5695 s->ofldtxq[i].q.size = 1024;
5696
5697 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5698 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5699
5700 init_rspq(&r->rspq, 0, 0, 1024, 64);
5701 r->rspq.uld = CXGB4_ULD_ISCSI;
5702 r->fl.size = 72;
5703 }
5704
5705 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5706 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5707
5708 init_rspq(&r->rspq, 0, 0, 511, 64);
5709 r->rspq.uld = CXGB4_ULD_RDMA;
5710 r->fl.size = 72;
5711 }
5712
5713 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
5714 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
5715 }
5716
5717 /*
5718 * Reduce the number of Ethernet queues across all ports to at most n.
5719 * n provides at least one queue per port.
5720 */
5721 static void reduce_ethqs(struct adapter *adap, int n)
5722 {
5723 int i;
5724 struct port_info *pi;
5725
5726 while (n < adap->sge.ethqsets)
5727 for_each_port(adap, i) {
5728 pi = adap2pinfo(adap, i);
5729 if (pi->nqsets > 1) {
5730 pi->nqsets--;
5731 adap->sge.ethqsets--;
5732 if (adap->sge.ethqsets <= n)
5733 break;
5734 }
5735 }
5736
5737 n = 0;
5738 for_each_port(adap, i) {
5739 pi = adap2pinfo(adap, i);
5740 pi->first_qset = n;
5741 n += pi->nqsets;
5742 }
5743 }
5744
5745 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5746 #define EXTRA_VECS 2
5747
5748 static int enable_msix(struct adapter *adap)
5749 {
5750 int ofld_need = 0;
5751 int i, want, need;
5752 struct sge *s = &adap->sge;
5753 unsigned int nchan = adap->params.nports;
5754 struct msix_entry entries[MAX_INGQ + 1];
5755
5756 for (i = 0; i < ARRAY_SIZE(entries); ++i)
5757 entries[i].entry = i;
5758
5759 want = s->max_ethqsets + EXTRA_VECS;
5760 if (is_offload(adap)) {
5761 want += s->rdmaqs + s->ofldqsets;
5762 /* need nchan for each possible ULD */
5763 ofld_need = 2 * nchan;
5764 }
5765 need = adap->params.nports + EXTRA_VECS + ofld_need;
5766
5767 want = pci_enable_msix_range(adap->pdev, entries, need, want);
5768 if (want < 0)
5769 return want;
5770
5771 /*
5772 * Distribute available vectors to the various queue groups.
5773 * Every group gets its minimum requirement and NIC gets top
5774 * priority for leftovers.
5775 */
5776 i = want - EXTRA_VECS - ofld_need;
5777 if (i < s->max_ethqsets) {
5778 s->max_ethqsets = i;
5779 if (i < s->ethqsets)
5780 reduce_ethqs(adap, i);
5781 }
5782 if (is_offload(adap)) {
5783 i = want - EXTRA_VECS - s->max_ethqsets;
5784 i -= ofld_need - nchan;
5785 s->ofldqsets = (i / nchan) * nchan; /* round down */
5786 }
5787 for (i = 0; i < want; ++i)
5788 adap->msix_info[i].vec = entries[i].vector;
5789
5790 return 0;
5791 }
5792
5793 #undef EXTRA_VECS
5794
5795 static int init_rss(struct adapter *adap)
5796 {
5797 unsigned int i, j;
5798
5799 for_each_port(adap, i) {
5800 struct port_info *pi = adap2pinfo(adap, i);
5801
5802 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5803 if (!pi->rss)
5804 return -ENOMEM;
5805 for (j = 0; j < pi->rss_size; j++)
5806 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
5807 }
5808 return 0;
5809 }
5810
5811 static void print_port_info(const struct net_device *dev)
5812 {
5813 char buf[80];
5814 char *bufp = buf;
5815 const char *spd = "";
5816 const struct port_info *pi = netdev_priv(dev);
5817 const struct adapter *adap = pi->adapter;
5818
5819 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5820 spd = " 2.5 GT/s";
5821 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5822 spd = " 5 GT/s";
5823
5824 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5825 bufp += sprintf(bufp, "100/");
5826 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5827 bufp += sprintf(bufp, "1000/");
5828 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5829 bufp += sprintf(bufp, "10G/");
5830 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5831 bufp += sprintf(bufp, "40G/");
5832 if (bufp != buf)
5833 --bufp;
5834 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
5835
5836 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
5837 adap->params.vpd.id,
5838 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
5839 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5840 (adap->flags & USING_MSIX) ? " MSI-X" :
5841 (adap->flags & USING_MSI) ? " MSI" : "");
5842 netdev_info(dev, "S/N: %s, E/C: %s\n",
5843 adap->params.vpd.sn, adap->params.vpd.ec);
5844 }
5845
5846 static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
5847 {
5848 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
5849 }
5850
5851 /*
5852 * Free the following resources:
5853 * - memory used for tables
5854 * - MSI/MSI-X
5855 * - net devices
5856 * - resources FW is holding for us
5857 */
5858 static void free_some_resources(struct adapter *adapter)
5859 {
5860 unsigned int i;
5861
5862 t4_free_mem(adapter->l2t);
5863 t4_free_mem(adapter->tids.tid_tab);
5864 disable_msi(adapter);
5865
5866 for_each_port(adapter, i)
5867 if (adapter->port[i]) {
5868 kfree(adap2pinfo(adapter, i)->rss);
5869 free_netdev(adapter->port[i]);
5870 }
5871 if (adapter->flags & FW_OK)
5872 t4_fw_bye(adapter, adapter->fn);
5873 }
5874
5875 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
5876 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
5877 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
5878 #define SEGMENT_SIZE 128
5879
5880 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5881 {
5882 int func, i, err, s_qpp, qpp, num_seg;
5883 struct port_info *pi;
5884 bool highdma = false;
5885 struct adapter *adapter = NULL;
5886
5887 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5888
5889 err = pci_request_regions(pdev, KBUILD_MODNAME);
5890 if (err) {
5891 /* Just info, some other driver may have claimed the device. */
5892 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
5893 return err;
5894 }
5895
5896 /* We control everything through one PF */
5897 func = PCI_FUNC(pdev->devfn);
5898 if (func != ent->driver_data) {
5899 pci_save_state(pdev); /* to restore SR-IOV later */
5900 goto sriov;
5901 }
5902
5903 err = pci_enable_device(pdev);
5904 if (err) {
5905 dev_err(&pdev->dev, "cannot enable PCI device\n");
5906 goto out_release_regions;
5907 }
5908
5909 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
5910 highdma = true;
5911 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5912 if (err) {
5913 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
5914 "coherent allocations\n");
5915 goto out_disable_device;
5916 }
5917 } else {
5918 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5919 if (err) {
5920 dev_err(&pdev->dev, "no usable DMA configuration\n");
5921 goto out_disable_device;
5922 }
5923 }
5924
5925 pci_enable_pcie_error_reporting(pdev);
5926 enable_pcie_relaxed_ordering(pdev);
5927 pci_set_master(pdev);
5928 pci_save_state(pdev);
5929
5930 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
5931 if (!adapter) {
5932 err = -ENOMEM;
5933 goto out_disable_device;
5934 }
5935
5936 /* PCI device has been enabled */
5937 adapter->flags |= DEV_ENABLED;
5938
5939 adapter->regs = pci_ioremap_bar(pdev, 0);
5940 if (!adapter->regs) {
5941 dev_err(&pdev->dev, "cannot map device registers\n");
5942 err = -ENOMEM;
5943 goto out_free_adapter;
5944 }
5945
5946 adapter->pdev = pdev;
5947 adapter->pdev_dev = &pdev->dev;
5948 adapter->mbox = func;
5949 adapter->fn = func;
5950 adapter->msg_enable = dflt_msg_enable;
5951 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
5952
5953 spin_lock_init(&adapter->stats_lock);
5954 spin_lock_init(&adapter->tid_release_lock);
5955
5956 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
5957 INIT_WORK(&adapter->db_full_task, process_db_full);
5958 INIT_WORK(&adapter->db_drop_task, process_db_drop);
5959
5960 err = t4_prep_adapter(adapter);
5961 if (err)
5962 goto out_unmap_bar0;
5963
5964 if (!is_t4(adapter->params.chip)) {
5965 s_qpp = QUEUESPERPAGEPF1 * adapter->fn;
5966 qpp = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter,
5967 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
5968 num_seg = PAGE_SIZE / SEGMENT_SIZE;
5969
5970 /* Each segment size is 128B. Write coalescing is enabled only
5971 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
5972 * queue is less no of segments that can be accommodated in
5973 * a page size.
5974 */
5975 if (qpp > num_seg) {
5976 dev_err(&pdev->dev,
5977 "Incorrect number of egress queues per page\n");
5978 err = -EINVAL;
5979 goto out_unmap_bar0;
5980 }
5981 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
5982 pci_resource_len(pdev, 2));
5983 if (!adapter->bar2) {
5984 dev_err(&pdev->dev, "cannot map device bar2 region\n");
5985 err = -ENOMEM;
5986 goto out_unmap_bar0;
5987 }
5988 }
5989
5990 setup_memwin(adapter);
5991 err = adap_init0(adapter);
5992 setup_memwin_rdma(adapter);
5993 if (err)
5994 goto out_unmap_bar;
5995
5996 for_each_port(adapter, i) {
5997 struct net_device *netdev;
5998
5999 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6000 MAX_ETH_QSETS);
6001 if (!netdev) {
6002 err = -ENOMEM;
6003 goto out_free_dev;
6004 }
6005
6006 SET_NETDEV_DEV(netdev, &pdev->dev);
6007
6008 adapter->port[i] = netdev;
6009 pi = netdev_priv(netdev);
6010 pi->adapter = adapter;
6011 pi->xact_addr_filt = -1;
6012 pi->port_id = i;
6013 netdev->irq = pdev->irq;
6014
6015 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6016 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6017 NETIF_F_RXCSUM | NETIF_F_RXHASH |
6018 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
6019 if (highdma)
6020 netdev->hw_features |= NETIF_F_HIGHDMA;
6021 netdev->features |= netdev->hw_features;
6022 netdev->vlan_features = netdev->features & VLAN_FEAT;
6023
6024 netdev->priv_flags |= IFF_UNICAST_FLT;
6025
6026 netdev->netdev_ops = &cxgb4_netdev_ops;
6027 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
6028 }
6029
6030 pci_set_drvdata(pdev, adapter);
6031
6032 if (adapter->flags & FW_OK) {
6033 err = t4_port_init(adapter, func, func, 0);
6034 if (err)
6035 goto out_free_dev;
6036 }
6037
6038 /*
6039 * Configure queues and allocate tables now, they can be needed as
6040 * soon as the first register_netdev completes.
6041 */
6042 cfg_queues(adapter);
6043
6044 adapter->l2t = t4_init_l2t();
6045 if (!adapter->l2t) {
6046 /* We tolerate a lack of L2T, giving up some functionality */
6047 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6048 adapter->params.offload = 0;
6049 }
6050
6051 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6052 dev_warn(&pdev->dev, "could not allocate TID table, "
6053 "continuing\n");
6054 adapter->params.offload = 0;
6055 }
6056
6057 /* See what interrupts we'll be using */
6058 if (msi > 1 && enable_msix(adapter) == 0)
6059 adapter->flags |= USING_MSIX;
6060 else if (msi > 0 && pci_enable_msi(pdev) == 0)
6061 adapter->flags |= USING_MSI;
6062
6063 err = init_rss(adapter);
6064 if (err)
6065 goto out_free_dev;
6066
6067 /*
6068 * The card is now ready to go. If any errors occur during device
6069 * registration we do not fail the whole card but rather proceed only
6070 * with the ports we manage to register successfully. However we must
6071 * register at least one net device.
6072 */
6073 for_each_port(adapter, i) {
6074 pi = adap2pinfo(adapter, i);
6075 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6076 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6077
6078 err = register_netdev(adapter->port[i]);
6079 if (err)
6080 break;
6081 adapter->chan_map[pi->tx_chan] = i;
6082 print_port_info(adapter->port[i]);
6083 }
6084 if (i == 0) {
6085 dev_err(&pdev->dev, "could not register any net devices\n");
6086 goto out_free_dev;
6087 }
6088 if (err) {
6089 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6090 err = 0;
6091 }
6092
6093 if (cxgb4_debugfs_root) {
6094 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6095 cxgb4_debugfs_root);
6096 setup_debugfs(adapter);
6097 }
6098
6099 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6100 pdev->needs_freset = 1;
6101
6102 if (is_offload(adapter))
6103 attach_ulds(adapter);
6104
6105 sriov:
6106 #ifdef CONFIG_PCI_IOV
6107 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
6108 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6109 dev_info(&pdev->dev,
6110 "instantiated %u virtual functions\n",
6111 num_vf[func]);
6112 #endif
6113 return 0;
6114
6115 out_free_dev:
6116 free_some_resources(adapter);
6117 out_unmap_bar:
6118 if (!is_t4(adapter->params.chip))
6119 iounmap(adapter->bar2);
6120 out_unmap_bar0:
6121 iounmap(adapter->regs);
6122 out_free_adapter:
6123 kfree(adapter);
6124 out_disable_device:
6125 pci_disable_pcie_error_reporting(pdev);
6126 pci_disable_device(pdev);
6127 out_release_regions:
6128 pci_release_regions(pdev);
6129 return err;
6130 }
6131
6132 static void remove_one(struct pci_dev *pdev)
6133 {
6134 struct adapter *adapter = pci_get_drvdata(pdev);
6135
6136 #ifdef CONFIG_PCI_IOV
6137 pci_disable_sriov(pdev);
6138
6139 #endif
6140
6141 if (adapter) {
6142 int i;
6143
6144 if (is_offload(adapter))
6145 detach_ulds(adapter);
6146
6147 for_each_port(adapter, i)
6148 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
6149 unregister_netdev(adapter->port[i]);
6150
6151 if (adapter->debugfs_root)
6152 debugfs_remove_recursive(adapter->debugfs_root);
6153
6154 /* If we allocated filters, free up state associated with any
6155 * valid filters ...
6156 */
6157 if (adapter->tids.ftid_tab) {
6158 struct filter_entry *f = &adapter->tids.ftid_tab[0];
6159 for (i = 0; i < (adapter->tids.nftids +
6160 adapter->tids.nsftids); i++, f++)
6161 if (f->valid)
6162 clear_filter(adapter, f);
6163 }
6164
6165 if (adapter->flags & FULL_INIT_DONE)
6166 cxgb_down(adapter);
6167
6168 free_some_resources(adapter);
6169 iounmap(adapter->regs);
6170 if (!is_t4(adapter->params.chip))
6171 iounmap(adapter->bar2);
6172 pci_disable_pcie_error_reporting(pdev);
6173 if ((adapter->flags & DEV_ENABLED)) {
6174 pci_disable_device(pdev);
6175 adapter->flags &= ~DEV_ENABLED;
6176 }
6177 pci_release_regions(pdev);
6178 kfree(adapter);
6179 } else
6180 pci_release_regions(pdev);
6181 }
6182
6183 static struct pci_driver cxgb4_driver = {
6184 .name = KBUILD_MODNAME,
6185 .id_table = cxgb4_pci_tbl,
6186 .probe = init_one,
6187 .remove = remove_one,
6188 .err_handler = &cxgb4_eeh,
6189 };
6190
6191 static int __init cxgb4_init_module(void)
6192 {
6193 int ret;
6194
6195 workq = create_singlethread_workqueue("cxgb4");
6196 if (!workq)
6197 return -ENOMEM;
6198
6199 /* Debugfs support is optional, just warn if this fails */
6200 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6201 if (!cxgb4_debugfs_root)
6202 pr_warn("could not create debugfs entry, continuing\n");
6203
6204 ret = pci_register_driver(&cxgb4_driver);
6205 if (ret < 0) {
6206 debugfs_remove(cxgb4_debugfs_root);
6207 destroy_workqueue(workq);
6208 }
6209
6210 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6211
6212 return ret;
6213 }
6214
6215 static void __exit cxgb4_cleanup_module(void)
6216 {
6217 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6218 pci_unregister_driver(&cxgb4_driver);
6219 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
6220 flush_workqueue(workq);
6221 destroy_workqueue(workq);
6222 }
6223
6224 module_init(cxgb4_init_module);
6225 module_exit(cxgb4_cleanup_module);
Linux kernel - Deliverables
This page took 0.560382 seconds and 6 git commands to generate.