projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
RDMA/nes: Use LRO
[deliverable/linux.git] / drivers / infiniband / hw / nes / nes_hw.c
1 /*
2 * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <linux/module.h>
35 #include <linux/moduleparam.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/ip.h>
39 #include <linux/tcp.h>
40 #include <linux/if_vlan.h>
41 #include <linux/inet_lro.h>
42
43 #include "nes.h"
44
45 static u32 crit_err_count;
46 u32 int_mod_timer_init;
47 u32 int_mod_cq_depth_256;
48 u32 int_mod_cq_depth_128;
49 u32 int_mod_cq_depth_32;
50 u32 int_mod_cq_depth_24;
51 u32 int_mod_cq_depth_16;
52 u32 int_mod_cq_depth_4;
53 u32 int_mod_cq_depth_1;
54
55 #include "nes_cm.h"
56
57 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
58 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
59 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
60 u8 OneG_Mode);
61 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
62 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
63 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
64 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
65 struct nes_hw_aeqe *aeqe);
66 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
67 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
68
69 #ifdef CONFIG_INFINIBAND_NES_DEBUG
70 static unsigned char *nes_iwarp_state_str[] = {
71 "Non-Existant",
72 "Idle",
73 "RTS",
74 "Closing",
75 "RSVD1",
76 "Terminate",
77 "Error",
78 "RSVD2",
79 };
80
81 static unsigned char *nes_tcp_state_str[] = {
82 "Non-Existant",
83 "Closed",
84 "Listen",
85 "SYN Sent",
86 "SYN Rcvd",
87 "Established",
88 "Close Wait",
89 "FIN Wait 1",
90 "Closing",
91 "Last Ack",
92 "FIN Wait 2",
93 "Time Wait",
94 "RSVD1",
95 "RSVD2",
96 "RSVD3",
97 "RSVD4",
98 };
99 #endif
100
101
102 /**
103 * nes_nic_init_timer_defaults
104 */
105 void nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
106 {
107 unsigned long flags;
108 struct nes_adapter *nesadapter = nesdev->nesadapter;
109 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
110
111 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
112
113 shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
114 shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
115 if (jumbomode) {
116 shared_timer->threshold_low = DEFAULT_JUMBO_NES_QL_LOW;
117 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
118 shared_timer->threshold_high = DEFAULT_JUMBO_NES_QL_HIGH;
119 } else {
120 shared_timer->threshold_low = DEFAULT_NES_QL_LOW;
121 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
122 shared_timer->threshold_high = DEFAULT_NES_QL_HIGH;
123 }
124
125 /* todo use netdev->mtu to set thresholds */
126 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
127 }
128
129
130 /**
131 * nes_nic_init_timer
132 */
133 static void nes_nic_init_timer(struct nes_device *nesdev)
134 {
135 unsigned long flags;
136 struct nes_adapter *nesadapter = nesdev->nesadapter;
137 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
138
139 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
140
141 if (shared_timer->timer_in_use_old == 0) {
142 nesdev->deepcq_count = 0;
143 shared_timer->timer_direction_upward = 0;
144 shared_timer->timer_direction_downward = 0;
145 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
146 shared_timer->timer_in_use_old = 0;
147
148 }
149 if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
150 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
151 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
152 0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
153 }
154 /* todo use netdev->mtu to set thresholds */
155 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
156 }
157
158
159 /**
160 * nes_nic_tune_timer
161 */
162 static void nes_nic_tune_timer(struct nes_device *nesdev)
163 {
164 unsigned long flags;
165 struct nes_adapter *nesadapter = nesdev->nesadapter;
166 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
167 u16 cq_count = nesdev->currcq_count;
168
169 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
170
171 if (shared_timer->cq_count_old <= cq_count)
172 shared_timer->cq_direction_downward = 0;
173 else
174 shared_timer->cq_direction_downward++;
175 shared_timer->cq_count_old = cq_count;
176 if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
177 if (cq_count <= shared_timer->threshold_low &&
178 shared_timer->threshold_low > 4) {
179 shared_timer->threshold_low = shared_timer->threshold_low/2;
180 shared_timer->cq_direction_downward=0;
181 nesdev->currcq_count = 0;
182 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
183 return;
184 }
185 }
186
187 if (cq_count > 1) {
188 nesdev->deepcq_count += cq_count;
189 if (cq_count <= shared_timer->threshold_low) { /* increase timer gently */
190 shared_timer->timer_direction_upward++;
191 shared_timer->timer_direction_downward = 0;
192 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
193 shared_timer->timer_direction_upward = 0;
194 shared_timer->timer_direction_downward = 0;
195 } else if (cq_count <= shared_timer->threshold_high) { /* decrease timer gently */
196 shared_timer->timer_direction_downward++;
197 shared_timer->timer_direction_upward = 0;
198 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
199 shared_timer->timer_in_use -= 2;
200 shared_timer->timer_direction_upward = 0;
201 shared_timer->timer_direction_downward++;
202 } else {
203 shared_timer->timer_in_use -= 4;
204 shared_timer->timer_direction_upward = 0;
205 shared_timer->timer_direction_downward++;
206 }
207
208 if (shared_timer->timer_direction_upward > 3 ) { /* using history */
209 shared_timer->timer_in_use += 3;
210 shared_timer->timer_direction_upward = 0;
211 shared_timer->timer_direction_downward = 0;
212 }
213 if (shared_timer->timer_direction_downward > 5) { /* using history */
214 shared_timer->timer_in_use -= 4 ;
215 shared_timer->timer_direction_downward = 0;
216 shared_timer->timer_direction_upward = 0;
217 }
218 }
219
220 /* boundary checking */
221 if (shared_timer->timer_in_use > NES_NIC_FAST_TIMER_HIGH)
222 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_HIGH;
223 else if (shared_timer->timer_in_use < NES_NIC_FAST_TIMER_LOW) {
224 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_LOW;
225 }
226
227 nesdev->currcq_count = 0;
228
229 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
230 }
231
232
233 /**
234 * nes_init_adapter - initialize adapter
235 */
236 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
237 struct nes_adapter *nesadapter = NULL;
238 unsigned long num_pds;
239 u32 u32temp;
240 u32 port_count;
241 u16 max_rq_wrs;
242 u16 max_sq_wrs;
243 u32 max_mr;
244 u32 max_256pbl;
245 u32 max_4kpbl;
246 u32 max_qp;
247 u32 max_irrq;
248 u32 max_cq;
249 u32 hte_index_mask;
250 u32 adapter_size;
251 u32 arp_table_size;
252 u16 vendor_id;
253 u8 OneG_Mode;
254 u8 func_index;
255
256 /* search the list of existing adapters */
257 list_for_each_entry(nesadapter, &nes_adapter_list, list) {
258 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
259 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
260 nesdev->pcidev->devfn,
261 PCI_SLOT(nesadapter->devfn),
262 nesadapter->bus_number,
263 PCI_SLOT(nesdev->pcidev->devfn),
264 nesdev->pcidev->bus->number );
265 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
266 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
267 nesadapter->ref_count++;
268 return nesadapter;
269 }
270 }
271
272 /* no adapter found */
273 num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
274 if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
275 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
276 hw_rev);
277 return NULL;
278 }
279
280 nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
281 nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
282 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
283 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
284 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
285
286 nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
287
288
289 if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
290 return NULL;
291 if (nes_init_serdes(nesdev, hw_rev, port_count, OneG_Mode))
292 return NULL;
293 nes_init_csr_ne020(nesdev, hw_rev, port_count);
294
295 max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
296 nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
297
298 u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
299 if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
300 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
301 max_qp, u32temp);
302 max_qp = (u32)1 << (u32temp & 0x001f);
303 }
304
305 hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
306 nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
307 max_qp, hte_index_mask);
308
309 u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
310
311 max_irrq = 1 << (u32temp & 0x001f);
312
313 if (max_qp > max_irrq) {
314 max_qp = max_irrq;
315 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
316 max_qp);
317 }
318
319 /* there should be no reason to allocate more pds than qps */
320 if (num_pds > max_qp)
321 num_pds = max_qp;
322
323 u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
324 max_mr = (u32)8192 << (u32temp & 0x7);
325
326 u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
327 max_256pbl = (u32)1 << (u32temp & 0x0000001f);
328 max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
329 max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
330
331 u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
332 arp_table_size = 1 << u32temp;
333
334 adapter_size = (sizeof(struct nes_adapter) +
335 (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
336 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
337 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
338 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
339 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
340 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
341 adapter_size += sizeof(struct nes_qp **) * max_qp;
342
343 /* allocate a new adapter struct */
344 nesadapter = kzalloc(adapter_size, GFP_KERNEL);
345 if (nesadapter == NULL) {
346 return NULL;
347 }
348
349 nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
350 nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
351
352 /* populate the new nesadapter */
353 nesadapter->devfn = nesdev->pcidev->devfn;
354 nesadapter->bus_number = nesdev->pcidev->bus->number;
355 nesadapter->ref_count = 1;
356 nesadapter->timer_int_req = 0xffff0000;
357 nesadapter->OneG_Mode = OneG_Mode;
358 nesadapter->doorbell_start = nesdev->doorbell_region;
359
360 /* nesadapter->tick_delta = clk_divisor; */
361 nesadapter->hw_rev = hw_rev;
362 nesadapter->port_count = port_count;
363
364 nesadapter->max_qp = max_qp;
365 nesadapter->hte_index_mask = hte_index_mask;
366 nesadapter->max_irrq = max_irrq;
367 nesadapter->max_mr = max_mr;
368 nesadapter->max_256pbl = max_256pbl - 1;
369 nesadapter->max_4kpbl = max_4kpbl - 1;
370 nesadapter->max_cq = max_cq;
371 nesadapter->free_256pbl = max_256pbl - 1;
372 nesadapter->free_4kpbl = max_4kpbl - 1;
373 nesadapter->max_pd = num_pds;
374 nesadapter->arp_table_size = arp_table_size;
375
376 nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
377 if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
378 nesadapter->et_use_adaptive_rx_coalesce = 0;
379 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
380 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
381 } else {
382 nesadapter->et_use_adaptive_rx_coalesce = 1;
383 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
384 nesadapter->et_rx_coalesce_usecs_irq = 0;
385 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
386 }
387 /* Setup and enable the periodic timer */
388 if (nesadapter->et_rx_coalesce_usecs_irq)
389 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
390 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
391 else
392 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
393
394 nesadapter->base_pd = 1;
395
396 nesadapter->device_cap_flags =
397 IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW;
398
399 nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
400 [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
401 nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
402 nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
403 nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
404 nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
405 nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
406
407
408 /* mark the usual suspect QPs and CQs as in use */
409 for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
410 set_bit(u32temp, nesadapter->allocated_qps);
411 set_bit(u32temp, nesadapter->allocated_cqs);
412 }
413
414 for (u32temp = 0; u32temp < 20; u32temp++)
415 set_bit(u32temp, nesadapter->allocated_pds);
416 u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
417
418 max_rq_wrs = ((u32temp >> 8) & 3);
419 switch (max_rq_wrs) {
420 case 0:
421 max_rq_wrs = 4;
422 break;
423 case 1:
424 max_rq_wrs = 16;
425 break;
426 case 2:
427 max_rq_wrs = 32;
428 break;
429 case 3:
430 max_rq_wrs = 512;
431 break;
432 }
433
434 max_sq_wrs = (u32temp & 3);
435 switch (max_sq_wrs) {
436 case 0:
437 max_sq_wrs = 4;
438 break;
439 case 1:
440 max_sq_wrs = 16;
441 break;
442 case 2:
443 max_sq_wrs = 32;
444 break;
445 case 3:
446 max_sq_wrs = 512;
447 break;
448 }
449 nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
450 nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
451
452 nesadapter->max_sge = 4;
453 nesadapter->max_cqe = 32767;
454
455 if (nes_read_eeprom_values(nesdev, nesadapter)) {
456 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
457 kfree(nesadapter);
458 return NULL;
459 }
460
461 u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
462 nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
463 (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
464
465 /* setup port configuration */
466 if (nesadapter->port_count == 1) {
467 u32temp = 0x00000000;
468 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
469 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
470 else
471 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
472 } else {
473 if (nesadapter->port_count == 2)
474 u32temp = 0x00000044;
475 else
476 u32temp = 0x000000e4;
477 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
478 }
479
480 nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, u32temp);
481 nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
482 nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
483
484 spin_lock_init(&nesadapter->resource_lock);
485 spin_lock_init(&nesadapter->phy_lock);
486 spin_lock_init(&nesadapter->pbl_lock);
487 spin_lock_init(&nesadapter->periodic_timer_lock);
488
489 INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
490 INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
491 INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
492 INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
493
494 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
495 u32 pcs_control_status0, pcs_control_status1;
496 u32 reset_value;
497 u32 i = 0;
498 u32 int_cnt = 0;
499 u32 ext_cnt = 0;
500 unsigned long flags;
501 u32 j = 0;
502
503 pcs_control_status0 = nes_read_indexed(nesdev,
504 NES_IDX_PHY_PCS_CONTROL_STATUS0);
505 pcs_control_status1 = nes_read_indexed(nesdev,
506 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
507
508 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
509 pcs_control_status0 = nes_read_indexed(nesdev,
510 NES_IDX_PHY_PCS_CONTROL_STATUS0);
511 pcs_control_status1 = nes_read_indexed(nesdev,
512 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
513 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
514 || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
515 int_cnt++;
516 msleep(1);
517 }
518 if (int_cnt > 1) {
519 spin_lock_irqsave(&nesadapter->phy_lock, flags);
520 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
521 mh_detected++;
522 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
523 reset_value |= 0x0000003d;
524 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
525
526 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
527 & 0x00000040) != 0x00000040) && (j++ < 5000));
528 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
529
530 pcs_control_status0 = nes_read_indexed(nesdev,
531 NES_IDX_PHY_PCS_CONTROL_STATUS0);
532 pcs_control_status1 = nes_read_indexed(nesdev,
533 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
534
535 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
536 pcs_control_status0 = nes_read_indexed(nesdev,
537 NES_IDX_PHY_PCS_CONTROL_STATUS0);
538 pcs_control_status1 = nes_read_indexed(nesdev,
539 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
540 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
541 || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
542 if (++ext_cnt > int_cnt) {
543 spin_lock_irqsave(&nesadapter->phy_lock, flags);
544 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
545 0x0000F0C8);
546 mh_detected++;
547 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
548 reset_value |= 0x0000003d;
549 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
550
551 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
552 & 0x00000040) != 0x00000040) && (j++ < 5000));
553 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
554 break;
555 }
556 }
557 msleep(1);
558 }
559 }
560 }
561
562 if (nesadapter->hw_rev == NE020_REV) {
563 init_timer(&nesadapter->mh_timer);
564 nesadapter->mh_timer.function = nes_mh_fix;
565 nesadapter->mh_timer.expires = jiffies + (HZ/5); /* 1 second */
566 nesadapter->mh_timer.data = (unsigned long)nesdev;
567 add_timer(&nesadapter->mh_timer);
568 } else {
569 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
570 }
571
572 init_timer(&nesadapter->lc_timer);
573 nesadapter->lc_timer.function = nes_clc;
574 nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */
575 nesadapter->lc_timer.data = (unsigned long)nesdev;
576 add_timer(&nesadapter->lc_timer);
577
578 list_add_tail(&nesadapter->list, &nes_adapter_list);
579
580 for (func_index = 0; func_index < 8; func_index++) {
581 pci_bus_read_config_word(nesdev->pcidev->bus,
582 PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
583 func_index), 0, &vendor_id);
584 if (vendor_id == 0xffff)
585 break;
586 }
587 nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
588 func_index, pci_name(nesdev->pcidev));
589 nesadapter->adapter_fcn_count = func_index;
590
591 return nesadapter;
592 }
593
594
595 /**
596 * nes_reset_adapter_ne020
597 */
598 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
599 {
600 u32 port_count;
601 u32 u32temp;
602 u32 i;
603
604 u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
605 port_count = ((u32temp & 0x00000300) >> 8) + 1;
606 /* TODO: assuming that both SERDES are set the same for now */
607 *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
608 nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
609 u32temp, port_count);
610 if (*OneG_Mode)
611 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
612 u32temp &= 0xff00ffc0;
613 switch (port_count) {
614 case 1:
615 u32temp |= 0x00ee0000;
616 break;
617 case 2:
618 u32temp |= 0x00cc0000;
619 break;
620 case 4:
621 u32temp |= 0x00000000;
622 break;
623 default:
624 return 0;
625 break;
626 }
627
628 /* check and do full reset if needed */
629 if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
630 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
631 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
632
633 i = 0;
634 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
635 mdelay(1);
636 if (i >= 10000) {
637 nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
638 return 0;
639 }
640
641 i = 0;
642 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
643 mdelay(1);
644 if (i >= 10000) {
645 printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
646 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
647 return 0;
648 }
649 }
650
651 /* port reset */
652 switch (port_count) {
653 case 1:
654 u32temp |= 0x00ee0010;
655 break;
656 case 2:
657 u32temp |= 0x00cc0030;
658 break;
659 case 4:
660 u32temp |= 0x00000030;
661 break;
662 }
663
664 nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
665 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
666
667 i = 0;
668 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
669 mdelay(1);
670 if (i >= 10000) {
671 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
672 return 0;
673 }
674
675 /* serdes 0 */
676 i = 0;
677 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
678 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
679 mdelay(1);
680 if (i >= 5000) {
681 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
682 return 0;
683 }
684
685 /* serdes 1 */
686 if (port_count > 1) {
687 i = 0;
688 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
689 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
690 mdelay(1);
691 if (i >= 5000) {
692 nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
693 return 0;
694 }
695 }
696
697 return port_count;
698 }
699
700
701 /**
702 * nes_init_serdes
703 */
704 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
705 u8 OneG_Mode)
706 {
707 int i;
708 u32 u32temp;
709
710 if (hw_rev != NE020_REV) {
711 /* init serdes 0 */
712
713 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
714 if (!OneG_Mode)
715 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
716 if (port_count > 1) {
717 /* init serdes 1 */
718 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
719 if (!OneG_Mode)
720 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
721 }
722 } else {
723 /* init serdes 0 */
724 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
725 i = 0;
726 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
727 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
728 mdelay(1);
729 if (i >= 5000) {
730 nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
731 return 1;
732 }
733 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
734 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
735 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
736 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
737 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
738 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
739 if (OneG_Mode)
740 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
741 else
742 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
743
744 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
745 if (port_count > 1) {
746 /* init serdes 1 */
747 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
748 i = 0;
749 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
750 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
751 mdelay(1);
752 if (i >= 5000) {
753 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
754 /* return 1; */
755 }
756 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
757 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
758 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
759 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
760 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
761 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
762 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
763 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
764 }
765 }
766 return 0;
767 }
768
769
770 /**
771 * nes_init_csr_ne020
772 * Initialize registers for ne020 hardware
773 */
774 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
775 {
776 u32 u32temp;
777
778 nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
779
780 nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
781 /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
782 nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
783 nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
784 /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
785 nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
786 nes_write_indexed(nesdev, 0x00000600, 0x55555555);
787 nes_write_indexed(nesdev, 0x00000604, 0x55555555);
788
789 /* TODO: move these MAC register settings to NIC bringup */
790 nes_write_indexed(nesdev, 0x00002000, 0x00000001);
791 nes_write_indexed(nesdev, 0x00002004, 0x00000001);
792 nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
793 nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
794 nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
795 nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
796 if (port_count > 1) {
797 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
798 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
799 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
800 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
801 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
802 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
803 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
804 }
805 if (port_count > 2) {
806 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
807 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
808 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
809 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
810 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
811 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
812 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
813
814 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
815 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
816 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
817 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
818 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
819 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
820 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
821 }
822
823 nes_write_indexed(nesdev, 0x00005000, 0x00018000);
824 /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
825 nes_write_indexed(nesdev, 0x00005004, 0x00020001);
826 nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
827 nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
828 nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
829 nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
830 nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
831
832 /* TODO: move this to code, get from EEPROM */
833 nes_write_indexed(nesdev, 0x00000900, 0x20000001);
834 nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
835 nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
836 //
837 nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
838 /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
839
840 if (hw_rev != NE020_REV) {
841 u32temp = nes_read_indexed(nesdev, 0x000008e8);
842 u32temp |= 0x80000000;
843 nes_write_indexed(nesdev, 0x000008e8, u32temp);
844 u32temp = nes_read_indexed(nesdev, 0x000021f8);
845 u32temp &= 0x7fffffff;
846 u32temp |= 0x7fff0010;
847 nes_write_indexed(nesdev, 0x000021f8, u32temp);
848 }
849 }
850
851
852 /**
853 * nes_destroy_adapter - destroy the adapter structure
854 */
855 void nes_destroy_adapter(struct nes_adapter *nesadapter)
856 {
857 struct nes_adapter *tmp_adapter;
858
859 list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
860 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
861 tmp_adapter);
862 }
863
864 nesadapter->ref_count--;
865 if (!nesadapter->ref_count) {
866 if (nesadapter->hw_rev == NE020_REV) {
867 del_timer(&nesadapter->mh_timer);
868 }
869 del_timer(&nesadapter->lc_timer);
870
871 list_del(&nesadapter->list);
872 kfree(nesadapter);
873 }
874 }
875
876
877 /**
878 * nes_init_cqp
879 */
880 int nes_init_cqp(struct nes_device *nesdev)
881 {
882 struct nes_adapter *nesadapter = nesdev->nesadapter;
883 struct nes_hw_cqp_qp_context *cqp_qp_context;
884 struct nes_hw_cqp_wqe *cqp_wqe;
885 struct nes_hw_ceq *ceq;
886 struct nes_hw_ceq *nic_ceq;
887 struct nes_hw_aeq *aeq;
888 void *vmem;
889 dma_addr_t pmem;
890 u32 count=0;
891 u32 cqp_head;
892 u64 u64temp;
893 u32 u32temp;
894
895 /* allocate CQP memory */
896 /* Need to add max_cq to the aeq size once cq overflow checking is added back */
897 /* SQ is 512 byte aligned, others are 256 byte aligned */
898 nesdev->cqp_mem_size = 512 +
899 (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
900 (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
901 max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
902 max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
903 (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
904 sizeof(struct nes_hw_cqp_qp_context);
905
906 nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
907 &nesdev->cqp_pbase);
908 if (!nesdev->cqp_vbase) {
909 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
910 return -ENOMEM;
911 }
912 memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
913
914 /* Allocate a twice the number of CQP requests as the SQ size */
915 nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
916 2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
917 if (nesdev->nes_cqp_requests == NULL) {
918 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
919 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
920 nesdev->cqp.sq_pbase);
921 return -ENOMEM;
922 }
923
924 nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
925 nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
926
927 spin_lock_init(&nesdev->cqp.lock);
928 init_waitqueue_head(&nesdev->cqp.waitq);
929
930 /* Setup Various Structures */
931 vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
932 ~(unsigned long)(512 - 1));
933 pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
934 ~(unsigned long long)(512 - 1));
935
936 nesdev->cqp.sq_vbase = vmem;
937 nesdev->cqp.sq_pbase = pmem;
938 nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
939 nesdev->cqp.sq_head = 0;
940 nesdev->cqp.sq_tail = 0;
941 nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
942
943 vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
944 pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
945
946 nesdev->ccq.cq_vbase = vmem;
947 nesdev->ccq.cq_pbase = pmem;
948 nesdev->ccq.cq_size = NES_CCQ_SIZE;
949 nesdev->ccq.cq_head = 0;
950 nesdev->ccq.ce_handler = nes_cqp_ce_handler;
951 nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
952
953 vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
954 pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
955
956 nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
957 ceq = &nesadapter->ceq[nesdev->ceq_index];
958 ceq->ceq_vbase = vmem;
959 ceq->ceq_pbase = pmem;
960 ceq->ceq_size = NES_CCEQ_SIZE;
961 ceq->ceq_head = 0;
962
963 vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
964 pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
965
966 nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
967 nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
968 nic_ceq->ceq_vbase = vmem;
969 nic_ceq->ceq_pbase = pmem;
970 nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
971 nic_ceq->ceq_head = 0;
972
973 vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
974 pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
975
976 aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
977 aeq->aeq_vbase = vmem;
978 aeq->aeq_pbase = pmem;
979 aeq->aeq_size = nesadapter->max_qp;
980 aeq->aeq_head = 0;
981
982 /* Setup QP Context */
983 vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
984 pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
985
986 cqp_qp_context = vmem;
987 cqp_qp_context->context_words[0] =
988 cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
989 cqp_qp_context->context_words[1] = 0;
990 cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
991 cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
992
993
994 /* Write the address to Create CQP */
995 if ((sizeof(dma_addr_t) > 4)) {
996 nes_write_indexed(nesdev,
997 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
998 ((u64)pmem) >> 32);
999 } else {
1000 nes_write_indexed(nesdev,
1001 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1002 }
1003 nes_write_indexed(nesdev,
1004 NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1005 (u32)pmem);
1006
1007 INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1008 INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1009
1010 for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1011 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1012 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1013 }
1014
1015 /* Write Create CCQ WQE */
1016 cqp_head = nesdev->cqp.sq_head++;
1017 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1018 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1019 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1020 (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1021 NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1022 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1023 (nesdev->ccq.cq_number |
1024 ((u32)nesdev->ceq_index << 16)));
1025 u64temp = (u64)nesdev->ccq.cq_pbase;
1026 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1027 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1028 u64temp = (unsigned long)&nesdev->ccq;
1029 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1030 cpu_to_le32((u32)(u64temp >> 1));
1031 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1032 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1033 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1034
1035 /* Write Create CEQ WQE */
1036 cqp_head = nesdev->cqp.sq_head++;
1037 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1038 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1039 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1040 (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1041 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1042 u64temp = (u64)ceq->ceq_pbase;
1043 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1044
1045 /* Write Create AEQ WQE */
1046 cqp_head = nesdev->cqp.sq_head++;
1047 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1048 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1049 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1050 (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1051 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1052 u64temp = (u64)aeq->aeq_pbase;
1053 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1054
1055 /* Write Create NIC CEQ WQE */
1056 cqp_head = nesdev->cqp.sq_head++;
1057 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1058 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1059 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1060 (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1061 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1062 u64temp = (u64)nic_ceq->ceq_pbase;
1063 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1064
1065 /* Poll until CCQP done */
1066 count = 0;
1067 do {
1068 if (count++ > 1000) {
1069 printk(KERN_ERR PFX "Error creating CQP\n");
1070 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1071 nesdev->cqp_vbase, nesdev->cqp_pbase);
1072 return -1;
1073 }
1074 udelay(10);
1075 } while (!(nes_read_indexed(nesdev,
1076 NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1077
1078 nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1079 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1080
1081 u32temp = 0x04800000;
1082 nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1083
1084 /* wait for the CCQ, CEQ, and AEQ to get created */
1085 count = 0;
1086 do {
1087 if (count++ > 1000) {
1088 printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1089 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1090 nesdev->cqp_vbase, nesdev->cqp_pbase);
1091 return -1;
1092 }
1093 udelay(10);
1094 } while (((nes_read_indexed(nesdev,
1095 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1096
1097 /* dump the QP status value */
1098 nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1099 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1100
1101 nesdev->cqp.sq_tail++;
1102
1103 return 0;
1104 }
1105
1106
1107 /**
1108 * nes_destroy_cqp
1109 */
1110 int nes_destroy_cqp(struct nes_device *nesdev)
1111 {
1112 struct nes_hw_cqp_wqe *cqp_wqe;
1113 u32 count = 0;
1114 u32 cqp_head;
1115 unsigned long flags;
1116
1117 do {
1118 if (count++ > 1000)
1119 break;
1120 udelay(10);
1121 } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1122
1123 /* Reset CCQ */
1124 nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1125 nesdev->ccq.cq_number);
1126
1127 /* Disable device interrupts */
1128 nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1129
1130 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1131
1132 /* Destroy the AEQ */
1133 cqp_head = nesdev->cqp.sq_head++;
1134 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1135 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1136 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1137 ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1138 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1139
1140 /* Destroy the NIC CEQ */
1141 cqp_head = nesdev->cqp.sq_head++;
1142 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1143 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1144 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1145 ((u32)nesdev->nic_ceq_index << 8));
1146
1147 /* Destroy the CEQ */
1148 cqp_head = nesdev->cqp.sq_head++;
1149 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1150 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1151 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1152 (nesdev->ceq_index << 8));
1153
1154 /* Destroy the CCQ */
1155 cqp_head = nesdev->cqp.sq_head++;
1156 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1157 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1158 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1159 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1160 ((u32)nesdev->ceq_index << 16));
1161
1162 /* Destroy CQP */
1163 cqp_head = nesdev->cqp.sq_head++;
1164 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1165 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1166 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1167 NES_CQP_QP_TYPE_CQP);
1168 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1169
1170 barrier();
1171 /* Ring doorbell (5 WQEs) */
1172 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1173
1174 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1175
1176 /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1177 count = 0;
1178 do {
1179 if (count++ > 1000) {
1180 printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1181 PCI_FUNC(nesdev->pcidev->devfn));
1182 break;
1183 }
1184 udelay(10);
1185 } while (((nes_read_indexed(nesdev,
1186 NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1187
1188 /* dump the QP status value */
1189 nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1190 PCI_FUNC(nesdev->pcidev->devfn),
1191 nes_read_indexed(nesdev,
1192 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1193
1194 kfree(nesdev->nes_cqp_requests);
1195
1196 /* Free the control structures */
1197 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1198 nesdev->cqp.sq_pbase);
1199
1200 return 0;
1201 }
1202
1203
1204 /**
1205 * nes_init_phy
1206 */
1207 int nes_init_phy(struct nes_device *nesdev)
1208 {
1209 struct nes_adapter *nesadapter = nesdev->nesadapter;
1210 u32 counter = 0;
1211 u32 mac_index = nesdev->mac_index;
1212 u32 tx_config;
1213 u16 phy_data;
1214
1215 if (nesadapter->OneG_Mode) {
1216 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1217 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1218 printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1219 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1220 tx_config |= 0x04;
1221 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1222 }
1223
1224 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1225 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1226 nesadapter->phy_index[mac_index], phy_data);
1227 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index], 0xb000);
1228
1229 /* Reset the PHY */
1230 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1231 udelay(100);
1232 counter = 0;
1233 do {
1234 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1235 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1236 if (counter++ > 100) break;
1237 } while (phy_data & 0x8000);
1238
1239 /* Setting no phy loopback */
1240 phy_data &= 0xbfff;
1241 phy_data |= 0x1140;
1242 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data);
1243 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1244 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1245
1246 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1247 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1248
1249 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1250 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1251
1252 /* Setting the interrupt mask */
1253 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1254 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1255 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1256
1257 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1258 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1259
1260 /* turning on flow control */
1261 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1262 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1263 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1264 (phy_data & ~(0x03E0)) | 0xc00);
1265 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1266 phy_data | 0xc00); */
1267 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1268 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1269
1270 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1271 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1272 /* Clear Half duplex */
1273 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1274 phy_data & ~(0x0100));
1275 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1276 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1277
1278 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1279 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1280 } else {
1281 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
1282 /* setup 10G MDIO operation */
1283 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1284 tx_config |= 0x14;
1285 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1286 }
1287 }
1288 return 0;
1289 }
1290
1291
1292 /**
1293 * nes_replenish_nic_rq
1294 */
1295 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1296 {
1297 unsigned long flags;
1298 dma_addr_t bus_address;
1299 struct sk_buff *skb;
1300 struct nes_hw_nic_rq_wqe *nic_rqe;
1301 struct nes_hw_nic *nesnic;
1302 struct nes_device *nesdev;
1303 u32 rx_wqes_posted = 0;
1304
1305 nesnic = &nesvnic->nic;
1306 nesdev = nesvnic->nesdev;
1307 spin_lock_irqsave(&nesnic->rq_lock, flags);
1308 if (nesnic->replenishing_rq !=0) {
1309 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1310 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1311 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1312 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1313 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
1314 add_timer(&nesvnic->rq_wqes_timer);
1315 } else
1316 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1317 return;
1318 }
1319 nesnic->replenishing_rq = 1;
1320 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1321 do {
1322 skb = dev_alloc_skb(nesvnic->max_frame_size);
1323 if (skb) {
1324 skb->dev = nesvnic->netdev;
1325
1326 bus_address = pci_map_single(nesdev->pcidev,
1327 skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1328
1329 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1330 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1331 cpu_to_le32(nesvnic->max_frame_size);
1332 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1333 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1334 cpu_to_le32((u32)bus_address);
1335 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1336 cpu_to_le32((u32)((u64)bus_address >> 32));
1337 nesnic->rx_skb[nesnic->rq_head] = skb;
1338 nesnic->rq_head++;
1339 nesnic->rq_head &= nesnic->rq_size - 1;
1340 atomic_dec(&nesvnic->rx_skbs_needed);
1341 barrier();
1342 if (++rx_wqes_posted == 255) {
1343 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1344 rx_wqes_posted = 0;
1345 }
1346 } else {
1347 spin_lock_irqsave(&nesnic->rq_lock, flags);
1348 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1349 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1350 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1351 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1352 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
1353 add_timer(&nesvnic->rq_wqes_timer);
1354 } else
1355 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1356 break;
1357 }
1358 } while (atomic_read(&nesvnic->rx_skbs_needed));
1359 barrier();
1360 if (rx_wqes_posted)
1361 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1362 nesnic->replenishing_rq = 0;
1363 }
1364
1365
1366 /**
1367 * nes_rq_wqes_timeout
1368 */
1369 static void nes_rq_wqes_timeout(unsigned long parm)
1370 {
1371 struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1372 printk("%s: Timer fired.\n", __func__);
1373 atomic_set(&nesvnic->rx_skb_timer_running, 0);
1374 if (atomic_read(&nesvnic->rx_skbs_needed))
1375 nes_replenish_nic_rq(nesvnic);
1376 }
1377
1378
1379 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1380 void **tcph, u64 *hdr_flags, void *priv)
1381 {
1382 unsigned int ip_len;
1383 struct iphdr *iph;
1384 skb_reset_network_header(skb);
1385 iph = ip_hdr(skb);
1386 if (iph->protocol != IPPROTO_TCP)
1387 return -1;
1388 ip_len = ip_hdrlen(skb);
1389 skb_set_transport_header(skb, ip_len);
1390 *tcph = tcp_hdr(skb);
1391
1392 *hdr_flags = LRO_IPV4 | LRO_TCP;
1393 *iphdr = iph;
1394 return 0;
1395 }
1396
1397
1398 /**
1399 * nes_init_nic_qp
1400 */
1401 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1402 {
1403 struct nes_hw_cqp_wqe *cqp_wqe;
1404 struct nes_hw_nic_sq_wqe *nic_sqe;
1405 struct nes_hw_nic_qp_context *nic_context;
1406 struct sk_buff *skb;
1407 struct nes_hw_nic_rq_wqe *nic_rqe;
1408 struct nes_vnic *nesvnic = netdev_priv(netdev);
1409 unsigned long flags;
1410 void *vmem;
1411 dma_addr_t pmem;
1412 u64 u64temp;
1413 int ret;
1414 u32 cqp_head;
1415 u32 counter;
1416 u32 wqe_count;
1417 u8 jumbomode=0;
1418
1419 /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1420 nesvnic->nic_mem_size = 256 +
1421 (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1422 (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1423 (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1424 (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1425 sizeof(struct nes_hw_nic_qp_context);
1426
1427 nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1428 &nesvnic->nic_pbase);
1429 if (!nesvnic->nic_vbase) {
1430 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1431 return -ENOMEM;
1432 }
1433 memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1434 nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1435 nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1436
1437 vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1438 ~(unsigned long)(256 - 1));
1439 pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1440 ~(unsigned long long)(256 - 1));
1441
1442 /* Setup the first Fragment buffers */
1443 nesvnic->nic.first_frag_vbase = vmem;
1444
1445 for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1446 nesvnic->nic.frag_paddr[counter] = pmem;
1447 pmem += sizeof(struct nes_first_frag);
1448 }
1449
1450 /* setup the SQ */
1451 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1452
1453 nesvnic->nic.sq_vbase = (void *)vmem;
1454 nesvnic->nic.sq_pbase = pmem;
1455 nesvnic->nic.sq_head = 0;
1456 nesvnic->nic.sq_tail = 0;
1457 nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1458 for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1459 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1460 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1461 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1462 NES_NIC_SQ_WQE_COMPLETION);
1463 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1464 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1465 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1466 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1467 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1468 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1469 }
1470
1471 nesvnic->get_cqp_request = nes_get_cqp_request;
1472 nesvnic->post_cqp_request = nes_post_cqp_request;
1473 nesvnic->mcrq_mcast_filter = NULL;
1474
1475 spin_lock_init(&nesvnic->nic.sq_lock);
1476 spin_lock_init(&nesvnic->nic.rq_lock);
1477
1478 /* setup the RQ */
1479 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1480 pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1481
1482
1483 nesvnic->nic.rq_vbase = vmem;
1484 nesvnic->nic.rq_pbase = pmem;
1485 nesvnic->nic.rq_head = 0;
1486 nesvnic->nic.rq_tail = 0;
1487 nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1488
1489 /* setup the CQ */
1490 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1491 pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1492
1493 if (nesdev->nesadapter->netdev_count > 2)
1494 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1495 else
1496 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1497
1498 nesvnic->nic_cq.cq_vbase = vmem;
1499 nesvnic->nic_cq.cq_pbase = pmem;
1500 nesvnic->nic_cq.cq_head = 0;
1501 nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1502
1503 nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1504
1505 /* Send CreateCQ request to CQP */
1506 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1507 cqp_head = nesdev->cqp.sq_head;
1508
1509 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1510 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1511
1512 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1513 NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1514 ((u32)nesvnic->nic_cq.cq_size << 16));
1515 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1516 nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1517 u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1518 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1519 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1520 u64temp = (unsigned long)&nesvnic->nic_cq;
1521 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1));
1522 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1523 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1524 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1525 if (++cqp_head >= nesdev->cqp.sq_size)
1526 cqp_head = 0;
1527 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1528 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1529
1530 /* Send CreateQP request to CQP */
1531 nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1532 nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1533 cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1534 ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1535 nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1536 nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1537 nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1538 if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1539 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1540 }
1541
1542 u64temp = (u64)nesvnic->nic.sq_pbase;
1543 nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1544 nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1545 u64temp = (u64)nesvnic->nic.rq_pbase;
1546 nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1547 nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1548
1549 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1550 NES_CQP_QP_TYPE_NIC);
1551 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1552 u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1553 (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1554 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1555
1556 if (++cqp_head >= nesdev->cqp.sq_size)
1557 cqp_head = 0;
1558 nesdev->cqp.sq_head = cqp_head;
1559
1560 barrier();
1561
1562 /* Ring doorbell (2 WQEs) */
1563 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1564
1565 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1566 nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1567 nesvnic->nic.qp_id);
1568
1569 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1570 NES_EVENT_TIMEOUT);
1571 nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1572 nesvnic->nic.qp_id, ret);
1573 if (!ret) {
1574 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1575 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1576 nesvnic->nic_pbase);
1577 return -EIO;
1578 }
1579
1580 /* Populate the RQ */
1581 for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1582 skb = dev_alloc_skb(nesvnic->max_frame_size);
1583 if (!skb) {
1584 nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1585
1586 nes_destroy_nic_qp(nesvnic);
1587 return -ENOMEM;
1588 }
1589
1590 skb->dev = netdev;
1591
1592 pmem = pci_map_single(nesdev->pcidev, skb->data,
1593 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1594
1595 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1596 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1597 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1598 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1599 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1600 nesvnic->nic.rx_skb[counter] = skb;
1601 }
1602
1603 wqe_count = NES_NIC_WQ_SIZE - 1;
1604 nesvnic->nic.rq_head = wqe_count;
1605 barrier();
1606 do {
1607 counter = min(wqe_count, ((u32)255));
1608 wqe_count -= counter;
1609 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1610 } while (wqe_count);
1611 init_timer(&nesvnic->rq_wqes_timer);
1612 nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1613 nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1614 nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1615 if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1616 {
1617 nes_nic_init_timer(nesdev);
1618 if (netdev->mtu > 1500)
1619 jumbomode = 1;
1620 nes_nic_init_timer_defaults(nesdev, jumbomode);
1621 }
1622 nesvnic->lro_mgr.max_aggr = NES_LRO_MAX_AGGR;
1623 nesvnic->lro_mgr.max_desc = NES_MAX_LRO_DESCRIPTORS;
1624 nesvnic->lro_mgr.lro_arr = nesvnic->lro_desc;
1625 nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1626 nesvnic->lro_mgr.features = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1627 nesvnic->lro_mgr.dev = netdev;
1628 nesvnic->lro_mgr.ip_summed = CHECKSUM_UNNECESSARY;
1629 nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1630 return 0;
1631 }
1632
1633
1634 /**
1635 * nes_destroy_nic_qp
1636 */
1637 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1638 {
1639 struct nes_device *nesdev = nesvnic->nesdev;
1640 struct nes_hw_cqp_wqe *cqp_wqe;
1641 struct nes_hw_nic_rq_wqe *nic_rqe;
1642 u64 wqe_frag;
1643 u32 cqp_head;
1644 unsigned long flags;
1645 int ret;
1646
1647 /* Free remaining NIC receive buffers */
1648 while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1649 nic_rqe = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1650 wqe_frag = (u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1651 wqe_frag |= ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
1652 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1653 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1654 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1655 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1656 }
1657
1658 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1659
1660 /* Destroy NIC QP */
1661 cqp_head = nesdev->cqp.sq_head;
1662 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1663 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1664
1665 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1666 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1667 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1668 nesvnic->nic.qp_id);
1669
1670 if (++cqp_head >= nesdev->cqp.sq_size)
1671 cqp_head = 0;
1672
1673 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1674
1675 /* Destroy NIC CQ */
1676 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1677 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1678 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1679 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1680 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1681
1682 if (++cqp_head >= nesdev->cqp.sq_size)
1683 cqp_head = 0;
1684
1685 nesdev->cqp.sq_head = cqp_head;
1686 barrier();
1687
1688 /* Ring doorbell (2 WQEs) */
1689 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1690
1691 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1692 nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1693 " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1694 cqp_head, nesdev->cqp.sq_head,
1695 nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1696
1697 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1698 NES_EVENT_TIMEOUT);
1699
1700 nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1701 " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1702 ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1703 if (!ret) {
1704 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1705 nesvnic->nic.qp_id);
1706 }
1707
1708 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1709 nesvnic->nic_pbase);
1710 }
1711
1712 /**
1713 * nes_napi_isr
1714 */
1715 int nes_napi_isr(struct nes_device *nesdev)
1716 {
1717 struct nes_adapter *nesadapter = nesdev->nesadapter;
1718 u32 int_stat;
1719
1720 if (nesdev->napi_isr_ran) {
1721 /* interrupt status has already been read in ISR */
1722 int_stat = nesdev->int_stat;
1723 } else {
1724 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1725 nesdev->int_stat = int_stat;
1726 nesdev->napi_isr_ran = 1;
1727 }
1728
1729 int_stat &= nesdev->int_req;
1730 /* iff NIC, process here, else wait for DPC */
1731 if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1732 nesdev->napi_isr_ran = 0;
1733 nes_write32(nesdev->regs+NES_INT_STAT,
1734 (int_stat &
1735 ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1736
1737 /* Process the CEQs */
1738 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1739
1740 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1741 (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1742 ((nesadapter->et_use_adaptive_rx_coalesce) &&
1743 (nesdev->deepcq_count > nesadapter->et_pkt_rate_low)))) ) {
1744 if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1745 /* Enable Periodic timer interrupts */
1746 nesdev->int_req |= NES_INT_TIMER;
1747 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1748 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1749 nes_write32(nesdev->regs+NES_TIMER_STAT,
1750 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1751 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1752 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1753 }
1754
1755 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1756 {
1757 nes_nic_init_timer(nesdev);
1758 }
1759 /* Enable interrupts, except CEQs */
1760 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1761 } else {
1762 /* Enable interrupts, make sure timer is off */
1763 nesdev->int_req &= ~NES_INT_TIMER;
1764 nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1765 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1766 }
1767 nesdev->deepcq_count = 0;
1768 return 1;
1769 } else {
1770 return 0;
1771 }
1772 }
1773
1774
1775 /**
1776 * nes_dpc
1777 */
1778 void nes_dpc(unsigned long param)
1779 {
1780 struct nes_device *nesdev = (struct nes_device *)param;
1781 struct nes_adapter *nesadapter = nesdev->nesadapter;
1782 u32 counter;
1783 u32 loop_counter = 0;
1784 u32 int_status_bit;
1785 u32 int_stat;
1786 u32 timer_stat;
1787 u32 temp_int_stat;
1788 u32 intf_int_stat;
1789 u32 debug_error;
1790 u32 processed_intf_int = 0;
1791 u16 processed_timer_int = 0;
1792 u16 completion_ints = 0;
1793 u16 timer_ints = 0;
1794
1795 /* nes_debug(NES_DBG_ISR, "\n"); */
1796
1797 do {
1798 timer_stat = 0;
1799 if (nesdev->napi_isr_ran) {
1800 nesdev->napi_isr_ran = 0;
1801 int_stat = nesdev->int_stat;
1802 } else
1803 int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
1804 if (processed_intf_int != 0)
1805 int_stat &= nesdev->int_req & ~NES_INT_INTF;
1806 else
1807 int_stat &= nesdev->int_req;
1808 if (processed_timer_int == 0) {
1809 processed_timer_int = 1;
1810 if (int_stat & NES_INT_TIMER) {
1811 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
1812 if ((timer_stat & nesdev->timer_int_req) == 0) {
1813 int_stat &= ~NES_INT_TIMER;
1814 }
1815 }
1816 } else {
1817 int_stat &= ~NES_INT_TIMER;
1818 }
1819
1820 if (int_stat) {
1821 if (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1822 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)) {
1823 /* Ack the interrupts */
1824 nes_write32(nesdev->regs+NES_INT_STAT,
1825 (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1826 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1827 }
1828
1829 temp_int_stat = int_stat;
1830 for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
1831 if (int_stat & int_status_bit) {
1832 nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
1833 temp_int_stat &= ~int_status_bit;
1834 completion_ints = 1;
1835 }
1836 if (!(temp_int_stat & 0x0000ffff))
1837 break;
1838 int_status_bit <<= 1;
1839 }
1840
1841 /* Process the AEQ for this pci function */
1842 int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
1843 if (int_stat & int_status_bit) {
1844 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
1845 }
1846
1847 /* Process the MAC interrupt for this pci function */
1848 int_status_bit = 1 << (24 + nesdev->mac_index);
1849 if (int_stat & int_status_bit) {
1850 nes_process_mac_intr(nesdev, nesdev->mac_index);
1851 }
1852
1853 if (int_stat & NES_INT_TIMER) {
1854 if (timer_stat & nesdev->timer_int_req) {
1855 nes_write32(nesdev->regs + NES_TIMER_STAT,
1856 (timer_stat & nesdev->timer_int_req) |
1857 ~(nesdev->nesadapter->timer_int_req));
1858 timer_ints = 1;
1859 }
1860 }
1861
1862 if (int_stat & NES_INT_INTF) {
1863 processed_intf_int = 1;
1864 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
1865 intf_int_stat &= nesdev->intf_int_req;
1866 if (NES_INTF_INT_CRITERR & intf_int_stat) {
1867 debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1868 printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1869 (u16)debug_error);
1870 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1871 0x01010000 | (debug_error & 0x0000ffff));
1872 /* BUG(); */
1873 if (crit_err_count++ > 10)
1874 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1875 }
1876 if (NES_INTF_INT_PCIERR & intf_int_stat) {
1877 printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
1878 BUG();
1879 }
1880 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
1881 printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
1882 BUG();
1883 }
1884 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
1885 }
1886
1887 if (int_stat & NES_INT_TSW) {
1888 }
1889 }
1890 /* Don't use the interface interrupt bit stay in loop */
1891 int_stat &= ~NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1892 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3;
1893 } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
1894
1895 if (timer_ints == 1) {
1896 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
1897 if (completion_ints == 0) {
1898 nesdev->timer_only_int_count++;
1899 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
1900 nesdev->timer_only_int_count = 0;
1901 nesdev->int_req &= ~NES_INT_TIMER;
1902 nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1903 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1904 } else {
1905 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1906 }
1907 } else {
1908 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1909 {
1910 nes_nic_init_timer(nesdev);
1911 }
1912 nesdev->timer_only_int_count = 0;
1913 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1914 }
1915 } else {
1916 nesdev->timer_only_int_count = 0;
1917 nesdev->int_req &= ~NES_INT_TIMER;
1918 nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1919 nes_write32(nesdev->regs+NES_TIMER_STAT,
1920 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1921 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1922 }
1923 } else {
1924 if ( (completion_ints == 1) &&
1925 (((nesadapter->et_rx_coalesce_usecs_irq) &&
1926 (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1927 ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
1928 (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
1929 /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
1930 nesdev->timer_only_int_count = 0;
1931 nesdev->int_req |= NES_INT_TIMER;
1932 nes_write32(nesdev->regs+NES_TIMER_STAT,
1933 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1934 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1935 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1936 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1937 } else {
1938 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1939 }
1940 }
1941 nesdev->deepcq_count = 0;
1942 }
1943
1944
1945 /**
1946 * nes_process_ceq
1947 */
1948 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
1949 {
1950 u64 u64temp;
1951 struct nes_hw_cq *cq;
1952 u32 head;
1953 u32 ceq_size;
1954
1955 /* nes_debug(NES_DBG_CQ, "\n"); */
1956 head = ceq->ceq_head;
1957 ceq_size = ceq->ceq_size;
1958
1959 do {
1960 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
1961 NES_CEQE_VALID) {
1962 u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX])))<<32) |
1963 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
1964 u64temp <<= 1;
1965 cq = *((struct nes_hw_cq **)&u64temp);
1966 /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
1967 barrier();
1968 ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
1969
1970 /* call the event handler */
1971 cq->ce_handler(nesdev, cq);
1972
1973 if (++head >= ceq_size)
1974 head = 0;
1975 } else {
1976 break;
1977 }
1978
1979 } while (1);
1980
1981 ceq->ceq_head = head;
1982 }
1983
1984
1985 /**
1986 * nes_process_aeq
1987 */
1988 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
1989 {
1990 // u64 u64temp;
1991 u32 head;
1992 u32 aeq_size;
1993 u32 aeqe_misc;
1994 u32 aeqe_cq_id;
1995 struct nes_hw_aeqe volatile *aeqe;
1996
1997 head = aeq->aeq_head;
1998 aeq_size = aeq->aeq_size;
1999
2000 do {
2001 aeqe = &aeq->aeq_vbase[head];
2002 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2003 break;
2004 aeqe_misc = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2005 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2006 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2007 if (aeqe_cq_id >= NES_FIRST_QPN) {
2008 /* dealing with an accelerated QP related AE */
2009 // u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])))<<32) |
2010 // ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2011 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2012 } else {
2013 /* TODO: dealing with a CQP related AE */
2014 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2015 (u16)(aeqe_misc >> 16));
2016 }
2017 }
2018
2019 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2020
2021 if (++head >= aeq_size)
2022 head = 0;
2023 }
2024 while (1);
2025 aeq->aeq_head = head;
2026 }
2027
2028 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2029 {
2030 struct nes_adapter *nesadapter = nesdev->nesadapter;
2031 u32 reset_value;
2032 u32 i=0;
2033 u32 u32temp;
2034
2035 if (nesadapter->hw_rev == NE020_REV) {
2036 return;
2037 }
2038 mh_detected++;
2039
2040 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2041
2042 if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2043 reset_value |= 0x0000001d;
2044 else
2045 reset_value |= 0x0000002d;
2046
2047 if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2048 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2049 nesadapter->link_interrupt_count[0] = 0;
2050 nesadapter->link_interrupt_count[1] = 0;
2051 u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2052 if (0x00000040 & u32temp)
2053 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2054 else
2055 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2056
2057 reset_value |= 0x0000003d;
2058 }
2059 nesadapter->link_interrupt_count[mac_index] = 0;
2060 }
2061
2062 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2063
2064 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2065 & 0x00000040) != 0x00000040) && (i++ < 5000));
2066
2067 if (0x0000003d == (reset_value & 0x0000003d)) {
2068 u32 pcs_control_status0, pcs_control_status1;
2069
2070 for (i = 0; i < 10; i++) {
2071 pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2072 pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2073 if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2074 && (pcs_control_status0 & 0x00100000))
2075 || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2076 && (pcs_control_status1 & 0x00100000)))
2077 continue;
2078 else
2079 break;
2080 }
2081 if (10 == i) {
2082 u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2083 if (0x00000040 & u32temp)
2084 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2085 else
2086 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2087
2088 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2089
2090 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2091 & 0x00000040) != 0x00000040) && (i++ < 5000));
2092 }
2093 }
2094 }
2095
2096 /**
2097 * nes_process_mac_intr
2098 */
2099 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2100 {
2101 unsigned long flags;
2102 u32 pcs_control_status;
2103 struct nes_adapter *nesadapter = nesdev->nesadapter;
2104 struct nes_vnic *nesvnic;
2105 u32 mac_status;
2106 u32 mac_index = nesdev->mac_index;
2107 u32 u32temp;
2108 u16 phy_data;
2109 u16 temp_phy_data;
2110
2111 spin_lock_irqsave(&nesadapter->phy_lock, flags);
2112 if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2113 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2114 return;
2115 }
2116 nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2117 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2118
2119 /* ack the MAC interrupt */
2120 mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2121 /* Clear the interrupt */
2122 nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2123
2124 nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2125
2126 if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2127 nesdev->link_status_interrupts++;
2128 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2129 spin_lock_irqsave(&nesadapter->phy_lock, flags);
2130 nes_reset_link(nesdev, mac_index);
2131 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2132 }
2133 /* read the PHY interrupt status register */
2134 if (nesadapter->OneG_Mode) {
2135 do {
2136 nes_read_1G_phy_reg(nesdev, 0x1a,
2137 nesadapter->phy_index[mac_index], &phy_data);
2138 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2139 nesadapter->phy_index[mac_index], phy_data);
2140 } while (phy_data&0x8000);
2141
2142 temp_phy_data = 0;
2143 do {
2144 nes_read_1G_phy_reg(nesdev, 0x11,
2145 nesadapter->phy_index[mac_index], &phy_data);
2146 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2147 nesadapter->phy_index[mac_index], phy_data);
2148 if (temp_phy_data == phy_data)
2149 break;
2150 temp_phy_data = phy_data;
2151 } while (1);
2152
2153 nes_read_1G_phy_reg(nesdev, 0x1e,
2154 nesadapter->phy_index[mac_index], &phy_data);
2155 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2156 nesadapter->phy_index[mac_index], phy_data);
2157
2158 nes_read_1G_phy_reg(nesdev, 1,
2159 nesadapter->phy_index[mac_index], &phy_data);
2160 nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2161 nesadapter->phy_index[mac_index], phy_data);
2162
2163 if (temp_phy_data & 0x1000) {
2164 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2165 phy_data = 4;
2166 } else {
2167 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2168 }
2169 }
2170 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2171 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2172 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2173 pcs_control_status = nes_read_indexed(nesdev,
2174 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2175 pcs_control_status = nes_read_indexed(nesdev,
2176 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2177 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2178 mac_index, pcs_control_status);
2179 if (nesadapter->OneG_Mode) {
2180 u32temp = 0x01010000;
2181 if (nesadapter->port_count > 2) {
2182 u32temp |= 0x02020000;
2183 }
2184 if ((pcs_control_status & u32temp)!= u32temp) {
2185 phy_data = 0;
2186 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2187 }
2188 } else if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
2189 nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2190 temp_phy_data = (u16)nes_read_indexed(nesdev,
2191 NES_IDX_MAC_MDIO_CONTROL);
2192 u32temp = 20;
2193 do {
2194 nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2195 phy_data = (u16)nes_read_indexed(nesdev,
2196 NES_IDX_MAC_MDIO_CONTROL);
2197 if ((phy_data == temp_phy_data) || (!(--u32temp)))
2198 break;
2199 temp_phy_data = phy_data;
2200 } while (1);
2201 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2202 __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2203
2204 } else {
2205 phy_data = (0x0f0f0000 == (pcs_control_status & 0x0f1f0000)) ? 4 : 0;
2206 }
2207
2208 if (phy_data & 0x0004) {
2209 nesadapter->mac_link_down[mac_index] = 0;
2210 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2211 nes_debug(NES_DBG_PHY, "The Link is UP!!. linkup was %d\n",
2212 nesvnic->linkup);
2213 if (nesvnic->linkup == 0) {
2214 printk(PFX "The Link is now up for port %u, netdev %p.\n",
2215 mac_index, nesvnic->netdev);
2216 if (netif_queue_stopped(nesvnic->netdev))
2217 netif_start_queue(nesvnic->netdev);
2218 nesvnic->linkup = 1;
2219 netif_carrier_on(nesvnic->netdev);
2220 }
2221 }
2222 } else {
2223 nesadapter->mac_link_down[mac_index] = 1;
2224 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2225 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2226 nesvnic->linkup);
2227 if (nesvnic->linkup == 1) {
2228 printk(PFX "The Link is now down for port %u, netdev %p.\n",
2229 mac_index, nesvnic->netdev);
2230 if (!(netif_queue_stopped(nesvnic->netdev)))
2231 netif_stop_queue(nesvnic->netdev);
2232 nesvnic->linkup = 0;
2233 netif_carrier_off(nesvnic->netdev);
2234 }
2235 }
2236 }
2237 }
2238
2239 nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2240 }
2241
2242
2243
2244 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2245 {
2246 struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2247
2248 netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2249 }
2250
2251
2252 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2253 * getting out of nic_ce_handler
2254 */
2255 #define MAX_RQES_TO_PROCESS 384
2256
2257 /**
2258 * nes_nic_ce_handler
2259 */
2260 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2261 {
2262 u64 u64temp;
2263 dma_addr_t bus_address;
2264 struct nes_hw_nic *nesnic;
2265 struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2266 struct nes_adapter *nesadapter = nesdev->nesadapter;
2267 struct nes_hw_nic_rq_wqe *nic_rqe;
2268 struct nes_hw_nic_sq_wqe *nic_sqe;
2269 struct sk_buff *skb;
2270 struct sk_buff *rx_skb;
2271 __le16 *wqe_fragment_length;
2272 u32 head;
2273 u32 cq_size;
2274 u32 rx_pkt_size;
2275 u32 cqe_count=0;
2276 u32 cqe_errv;
2277 u32 cqe_misc;
2278 u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */
2279 u16 vlan_tag;
2280 u16 pkt_type;
2281 u16 rqes_processed = 0;
2282 u8 sq_cqes = 0;
2283 u8 nes_use_lro = 0;
2284
2285 head = cq->cq_head;
2286 cq_size = cq->cq_size;
2287 cq->cqes_pending = 1;
2288 if (nesvnic->netdev->features & NETIF_F_LRO)
2289 nes_use_lro = 1;
2290 do {
2291 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2292 NES_NIC_CQE_VALID) {
2293 nesnic = &nesvnic->nic;
2294 cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2295 if (cqe_misc & NES_NIC_CQE_SQ) {
2296 sq_cqes++;
2297 wqe_fragment_index = 1;
2298 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2299 skb = nesnic->tx_skb[nesnic->sq_tail];
2300 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2301 /* bump past the vlan tag */
2302 wqe_fragment_length++;
2303 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2304 u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2305 u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2306 bus_address = (dma_addr_t)u64temp;
2307 if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2308 pci_unmap_single(nesdev->pcidev,
2309 bus_address,
2310 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2311 PCI_DMA_TODEVICE);
2312 }
2313 for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2314 if (wqe_fragment_length[wqe_fragment_index]) {
2315 u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2316 u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2317 bus_address = (dma_addr_t)u64temp;
2318 pci_unmap_page(nesdev->pcidev,
2319 bus_address,
2320 le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2321 PCI_DMA_TODEVICE);
2322 } else
2323 break;
2324 }
2325 if (skb)
2326 dev_kfree_skb_any(skb);
2327 }
2328 nesnic->sq_tail++;
2329 nesnic->sq_tail &= nesnic->sq_size-1;
2330 if (sq_cqes > 128) {
2331 barrier();
2332 /* restart the queue if it had been stopped */
2333 if (netif_queue_stopped(nesvnic->netdev))
2334 netif_wake_queue(nesvnic->netdev);
2335 sq_cqes = 0;
2336 }
2337 } else {
2338 rqes_processed ++;
2339
2340 cq->rx_cqes_completed++;
2341 cq->rx_pkts_indicated++;
2342 rx_pkt_size = cqe_misc & 0x0000ffff;
2343 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2344 /* Get the skb */
2345 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2346 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2347 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2348 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2349 pci_unmap_single(nesdev->pcidev, bus_address,
2350 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2351 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2352 /* rx_skb->len = rx_pkt_size; */
2353 rx_skb->len = 0; /* TODO: see if this is necessary */
2354 skb_put(rx_skb, rx_pkt_size);
2355 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2356 nesnic->rq_tail++;
2357 nesnic->rq_tail &= nesnic->rq_size - 1;
2358
2359 atomic_inc(&nesvnic->rx_skbs_needed);
2360 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2361 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2362 cq->cq_number | (cqe_count << 16));
2363 // nesadapter->tune_timer.cq_count += cqe_count;
2364 nesdev->currcq_count += cqe_count;
2365 cqe_count = 0;
2366 nes_replenish_nic_rq(nesvnic);
2367 }
2368 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2369 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2370 rx_skb->ip_summed = CHECKSUM_NONE;
2371
2372 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2373 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2374 if ((cqe_errv &
2375 (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2376 NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2377 if (nesvnic->rx_checksum_disabled == 0) {
2378 rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2379 }
2380 } else
2381 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2382 " errv = 0x%X, pkt_type = 0x%X.\n",
2383 nesvnic->netdev->name, cqe_errv, pkt_type);
2384
2385 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2386 if ((cqe_errv &
2387 (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2388 NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2389 if (nesvnic->rx_checksum_disabled == 0) {
2390 rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2391 /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2392 nesvnic->netdev->name); */
2393 }
2394 } else
2395 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2396 " errv = 0x%X, pkt_type = 0x%X.\n",
2397 nesvnic->netdev->name, cqe_errv, pkt_type);
2398 }
2399 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2400 pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2401
2402 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2403 nes_cm_recv(rx_skb, nesvnic->netdev);
2404 } else {
2405 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2406 vlan_tag = (u16)(le32_to_cpu(
2407 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2408 >> 16);
2409 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2410 nesvnic->netdev->name, vlan_tag);
2411 if (nes_use_lro)
2412 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2413 nesvnic->vlan_grp, vlan_tag, NULL);
2414 else
2415 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2416 } else {
2417 if (nes_use_lro)
2418 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2419 else
2420 nes_netif_rx(rx_skb);
2421 }
2422 }
2423
2424 nesvnic->netdev->last_rx = jiffies;
2425 /* nesvnic->netstats.rx_packets++; */
2426 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2427 }
2428
2429 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2430 /* Accounting... */
2431 cqe_count++;
2432 if (++head >= cq_size)
2433 head = 0;
2434 if (cqe_count == 255) {
2435 /* Replenish Nic CQ */
2436 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2437 cq->cq_number | (cqe_count << 16));
2438 // nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2439 nesdev->currcq_count += cqe_count;
2440 cqe_count = 0;
2441 }
2442
2443 if (cq->rx_cqes_completed >= nesvnic->budget)
2444 break;
2445 } else {
2446 cq->cqes_pending = 0;
2447 break;
2448 }
2449
2450 } while (1);
2451
2452 if (nes_use_lro)
2453 lro_flush_all(&nesvnic->lro_mgr);
2454 if (sq_cqes) {
2455 barrier();
2456 /* restart the queue if it had been stopped */
2457 if (netif_queue_stopped(nesvnic->netdev))
2458 netif_wake_queue(nesvnic->netdev);
2459 }
2460 cq->cq_head = head;
2461 /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2462 cq->cq_number, cqe_count, cq->cq_head); */
2463 cq->cqe_allocs_pending = cqe_count;
2464 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2465 {
2466 // nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2467 nesdev->currcq_count += cqe_count;
2468 nes_nic_tune_timer(nesdev);
2469 }
2470 if (atomic_read(&nesvnic->rx_skbs_needed))
2471 nes_replenish_nic_rq(nesvnic);
2472 }
2473
2474
2475 /**
2476 * nes_cqp_ce_handler
2477 */
2478 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2479 {
2480 u64 u64temp;
2481 unsigned long flags;
2482 struct nes_hw_cqp *cqp = NULL;
2483 struct nes_cqp_request *cqp_request;
2484 struct nes_hw_cqp_wqe *cqp_wqe;
2485 u32 head;
2486 u32 cq_size;
2487 u32 cqe_count=0;
2488 u32 error_code;
2489 /* u32 counter; */
2490
2491 head = cq->cq_head;
2492 cq_size = cq->cq_size;
2493
2494 do {
2495 /* process the CQE */
2496 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2497 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2498
2499 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2500 u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2501 cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) |
2502 ((u64)(le32_to_cpu(cq->cq_vbase[head].
2503 cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2504 cqp = *((struct nes_hw_cqp **)&u64temp);
2505
2506 error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2507 if (error_code) {
2508 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2509 " Major/Minor codes = 0x%04X:%04X.\n",
2510 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2511 (u16)(error_code >> 16),
2512 (u16)error_code);
2513 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2514 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2515 }
2516
2517 u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2518 wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX])))<<32) |
2519 ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2520 wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2521 cqp_request = *((struct nes_cqp_request **)&u64temp);
2522 if (cqp_request) {
2523 if (cqp_request->waiting) {
2524 /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2525 cqp_request->major_code = (u16)(error_code >> 16);
2526 cqp_request->minor_code = (u16)error_code;
2527 barrier();
2528 cqp_request->request_done = 1;
2529 wake_up(&cqp_request->waitq);
2530 if (atomic_dec_and_test(&cqp_request->refcount)) {
2531 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2532 cqp_request,
2533 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2534 if (cqp_request->dynamic) {
2535 kfree(cqp_request);
2536 } else {
2537 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2538 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2539 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2540 }
2541 }
2542 } else if (cqp_request->callback) {
2543 /* Envoke the callback routine */
2544 cqp_request->cqp_callback(nesdev, cqp_request);
2545 if (cqp_request->dynamic) {
2546 kfree(cqp_request);
2547 } else {
2548 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2549 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2550 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2551 }
2552 } else {
2553 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2554 cqp_request,
2555 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2556 if (cqp_request->dynamic) {
2557 kfree(cqp_request);
2558 } else {
2559 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2560 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2561 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2562 }
2563 }
2564 } else {
2565 wake_up(&nesdev->cqp.waitq);
2566 }
2567
2568 cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2569 nes_write32(nesdev->regs+NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2570 if (++cqp->sq_tail >= cqp->sq_size)
2571 cqp->sq_tail = 0;
2572
2573 /* Accounting... */
2574 cqe_count++;
2575 if (++head >= cq_size)
2576 head = 0;
2577 } else {
2578 break;
2579 }
2580 } while (1);
2581 cq->cq_head = head;
2582
2583 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2584 while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2585 ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2586 (nesdev->cqp.sq_size - 1)) != 1)) {
2587 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2588 struct nes_cqp_request, list);
2589 list_del_init(&cqp_request->list);
2590 head = nesdev->cqp.sq_head++;
2591 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2592 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2593 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2594 barrier();
2595 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2596 cpu_to_le32((u32)((unsigned long)cqp_request));
2597 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2598 cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2599 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2600 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2601 /* Ring doorbell (1 WQEs) */
2602 barrier();
2603 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2604 }
2605 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2606
2607 /* Arm the CCQ */
2608 nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2609 cq->cq_number);
2610 nes_read32(nesdev->regs+NES_CQE_ALLOC);
2611 }
2612
2613
2614 /**
2615 * nes_process_iwarp_aeqe
2616 */
2617 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2618 struct nes_hw_aeqe *aeqe)
2619 {
2620 u64 context;
2621 u64 aeqe_context = 0;
2622 unsigned long flags;
2623 struct nes_qp *nesqp;
2624 int resource_allocated;
2625 /* struct iw_cm_id *cm_id; */
2626 struct nes_adapter *nesadapter = nesdev->nesadapter;
2627 struct ib_event ibevent;
2628 /* struct iw_cm_event cm_event; */
2629 u32 aeq_info;
2630 u32 next_iwarp_state = 0;
2631 u16 async_event_id;
2632 u8 tcp_state;
2633 u8 iwarp_state;
2634
2635 nes_debug(NES_DBG_AEQ, "\n");
2636 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2637 if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2638 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2639 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2640 } else {
2641 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2642 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2643 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2644 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2645 BUG_ON(!context);
2646 }
2647
2648 async_event_id = (u16)aeq_info;
2649 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2650 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2651 nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2652 " Tcp state = %s, iWARP state = %s\n",
2653 async_event_id,
2654 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2655 nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2656
2657
2658 switch (async_event_id) {
2659 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2660 nesqp = *((struct nes_qp **)&context);
2661 if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2662 nesqp->cm_id->add_ref(nesqp->cm_id);
2663 nes_add_ref(&nesqp->ibqp);
2664 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2665 NES_TIMER_TYPE_CLOSE, 1, 0);
2666 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2667 " need ae to finish up, original_last_aeq = 0x%04X."
2668 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2669 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2670 async_event_id, nesqp->last_aeq, tcp_state);
2671 }
2672 if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2673 (nesqp->ibqp_state != IB_QPS_RTS)) {
2674 /* FIN Received but tcp state or IB state moved on,
2675 should expect a close complete */
2676 return;
2677 }
2678 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2679 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2680 case NES_AEQE_AEID_TERMINATE_SENT:
2681 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2682 case NES_AEQE_AEID_RESET_SENT:
2683 nesqp = *((struct nes_qp **)&context);
2684 if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2685 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2686 }
2687 nes_add_ref(&nesqp->ibqp);
2688 spin_lock_irqsave(&nesqp->lock, flags);
2689 nesqp->hw_iwarp_state = iwarp_state;
2690 nesqp->hw_tcp_state = tcp_state;
2691 nesqp->last_aeq = async_event_id;
2692
2693 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2694 (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2695 nesqp->hte_added = 0;
2696 spin_unlock_irqrestore(&nesqp->lock, flags);
2697 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2698 nesqp->hwqp.qp_id);
2699 nes_hw_modify_qp(nesdev, nesqp,
2700 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2701 spin_lock_irqsave(&nesqp->lock, flags);
2702 }
2703
2704 if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2705 ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2706 (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2707 switch (nesqp->hw_iwarp_state) {
2708 case NES_AEQE_IWARP_STATE_RTS:
2709 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2710 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2711 break;
2712 case NES_AEQE_IWARP_STATE_TERMINATE:
2713 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2714 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2715 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2716 next_iwarp_state |= 0x02000000;
2717 nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2718 }
2719 break;
2720 default:
2721 next_iwarp_state = 0;
2722 }
2723 spin_unlock_irqrestore(&nesqp->lock, flags);
2724 if (next_iwarp_state) {
2725 nes_add_ref(&nesqp->ibqp);
2726 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2727 " also added another reference\n",
2728 nesqp->hwqp.qp_id, next_iwarp_state);
2729 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2730 }
2731 nes_cm_disconn(nesqp);
2732 } else {
2733 if (async_event_id == NES_AEQE_AEID_LLP_FIN_RECEIVED) {
2734 /* FIN Received but ib state not RTS,
2735 close complete will be on its way */
2736 spin_unlock_irqrestore(&nesqp->lock, flags);
2737 nes_rem_ref(&nesqp->ibqp);
2738 return;
2739 }
2740 spin_unlock_irqrestore(&nesqp->lock, flags);
2741 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2742 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
2743 nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2744 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2745 " also added another reference\n",
2746 nesqp->hwqp.qp_id, next_iwarp_state);
2747 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2748 }
2749 nes_cm_disconn(nesqp);
2750 }
2751 break;
2752 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
2753 nesqp = *((struct nes_qp **)&context);
2754 spin_lock_irqsave(&nesqp->lock, flags);
2755 nesqp->hw_iwarp_state = iwarp_state;
2756 nesqp->hw_tcp_state = tcp_state;
2757 nesqp->last_aeq = async_event_id;
2758 spin_unlock_irqrestore(&nesqp->lock, flags);
2759 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
2760 " event on QP%u \n Q2 Data:\n",
2761 nesqp->hwqp.qp_id);
2762 if (nesqp->ibqp.event_handler) {
2763 ibevent.device = nesqp->ibqp.device;
2764 ibevent.element.qp = &nesqp->ibqp;
2765 ibevent.event = IB_EVENT_QP_FATAL;
2766 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2767 }
2768 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2769 ((nesqp->ibqp_state == IB_QPS_RTS)&&
2770 (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2771 nes_add_ref(&nesqp->ibqp);
2772 nes_cm_disconn(nesqp);
2773 } else {
2774 nesqp->in_disconnect = 0;
2775 wake_up(&nesqp->kick_waitq);
2776 }
2777 break;
2778 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
2779 nesqp = *((struct nes_qp **)&context);
2780 nes_add_ref(&nesqp->ibqp);
2781 spin_lock_irqsave(&nesqp->lock, flags);
2782 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
2783 nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2784 nesqp->last_aeq = async_event_id;
2785 if (nesqp->cm_id) {
2786 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2787 " event on QP%u, remote IP = 0x%08X \n",
2788 nesqp->hwqp.qp_id,
2789 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
2790 } else {
2791 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2792 " event on QP%u \n",
2793 nesqp->hwqp.qp_id);
2794 }
2795 spin_unlock_irqrestore(&nesqp->lock, flags);
2796 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
2797 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2798 if (nesqp->ibqp.event_handler) {
2799 ibevent.device = nesqp->ibqp.device;
2800 ibevent.element.qp = &nesqp->ibqp;
2801 ibevent.event = IB_EVENT_QP_FATAL;
2802 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2803 }
2804 break;
2805 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
2806 if (NES_AEQE_INBOUND_RDMA&aeq_info) {
2807 nesqp = nesadapter->qp_table[le32_to_cpu(
2808 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2809 } else {
2810 /* TODO: get the actual WQE and mask off wqe index */
2811 context &= ~((u64)511);
2812 nesqp = *((struct nes_qp **)&context);
2813 }
2814 spin_lock_irqsave(&nesqp->lock, flags);
2815 nesqp->hw_iwarp_state = iwarp_state;
2816 nesqp->hw_tcp_state = tcp_state;
2817 nesqp->last_aeq = async_event_id;
2818 spin_unlock_irqrestore(&nesqp->lock, flags);
2819 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
2820 nesqp->hwqp.qp_id);
2821 if (nesqp->ibqp.event_handler) {
2822 ibevent.device = nesqp->ibqp.device;
2823 ibevent.element.qp = &nesqp->ibqp;
2824 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2825 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2826 }
2827 break;
2828 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
2829 nesqp = *((struct nes_qp **)&context);
2830 spin_lock_irqsave(&nesqp->lock, flags);
2831 nesqp->hw_iwarp_state = iwarp_state;
2832 nesqp->hw_tcp_state = tcp_state;
2833 nesqp->last_aeq = async_event_id;
2834 spin_unlock_irqrestore(&nesqp->lock, flags);
2835 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
2836 nesqp->hwqp.qp_id);
2837 if (nesqp->ibqp.event_handler) {
2838 ibevent.device = nesqp->ibqp.device;
2839 ibevent.element.qp = &nesqp->ibqp;
2840 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2841 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2842 }
2843 break;
2844 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
2845 nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
2846 [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2847 spin_lock_irqsave(&nesqp->lock, flags);
2848 nesqp->hw_iwarp_state = iwarp_state;
2849 nesqp->hw_tcp_state = tcp_state;
2850 nesqp->last_aeq = async_event_id;
2851 spin_unlock_irqrestore(&nesqp->lock, flags);
2852 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
2853 " nesqp = %p, AE reported %p\n",
2854 nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
2855 if (nesqp->ibqp.event_handler) {
2856 ibevent.device = nesqp->ibqp.device;
2857 ibevent.element.qp = &nesqp->ibqp;
2858 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2859 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2860 }
2861 break;
2862 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
2863 context <<= 1;
2864 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
2865 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
2866 resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
2867 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2868 if (resource_allocated) {
2869 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
2870 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2871 }
2872 break;
2873 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
2874 nesqp = nesadapter->qp_table[le32_to_cpu(
2875 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2876 spin_lock_irqsave(&nesqp->lock, flags);
2877 nesqp->hw_iwarp_state = iwarp_state;
2878 nesqp->hw_tcp_state = tcp_state;
2879 nesqp->last_aeq = async_event_id;
2880 spin_unlock_irqrestore(&nesqp->lock, flags);
2881 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
2882 "_FOR_AVAILABLE_BUFFER event on QP%u\n",
2883 nesqp->hwqp.qp_id);
2884 if (nesqp->ibqp.event_handler) {
2885 ibevent.device = nesqp->ibqp.device;
2886 ibevent.element.qp = &nesqp->ibqp;
2887 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2888 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2889 }
2890 /* tell cm to disconnect, cm will queue work to thread */
2891 nes_add_ref(&nesqp->ibqp);
2892 nes_cm_disconn(nesqp);
2893 break;
2894 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
2895 nesqp = *((struct nes_qp **)&context);
2896 spin_lock_irqsave(&nesqp->lock, flags);
2897 nesqp->hw_iwarp_state = iwarp_state;
2898 nesqp->hw_tcp_state = tcp_state;
2899 nesqp->last_aeq = async_event_id;
2900 spin_unlock_irqrestore(&nesqp->lock, flags);
2901 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
2902 "_NO_BUFFER_AVAILABLE event on QP%u\n",
2903 nesqp->hwqp.qp_id);
2904 if (nesqp->ibqp.event_handler) {
2905 ibevent.device = nesqp->ibqp.device;
2906 ibevent.element.qp = &nesqp->ibqp;
2907 ibevent.event = IB_EVENT_QP_FATAL;
2908 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2909 }
2910 /* tell cm to disconnect, cm will queue work to thread */
2911 nes_add_ref(&nesqp->ibqp);
2912 nes_cm_disconn(nesqp);
2913 break;
2914 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
2915 nesqp = *((struct nes_qp **)&context);
2916 spin_lock_irqsave(&nesqp->lock, flags);
2917 nesqp->hw_iwarp_state = iwarp_state;
2918 nesqp->hw_tcp_state = tcp_state;
2919 nesqp->last_aeq = async_event_id;
2920 spin_unlock_irqrestore(&nesqp->lock, flags);
2921 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
2922 " event on QP%u \n Q2 Data:\n",
2923 nesqp->hwqp.qp_id);
2924 if (nesqp->ibqp.event_handler) {
2925 ibevent.device = nesqp->ibqp.device;
2926 ibevent.element.qp = &nesqp->ibqp;
2927 ibevent.event = IB_EVENT_QP_FATAL;
2928 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2929 }
2930 /* tell cm to disconnect, cm will queue work to thread */
2931 nes_add_ref(&nesqp->ibqp);
2932 nes_cm_disconn(nesqp);
2933 break;
2934 /* TODO: additional AEs need to be here */
2935 default:
2936 nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
2937 async_event_id);
2938 break;
2939 }
2940
2941 }
2942
2943
2944 /**
2945 * nes_iwarp_ce_handler
2946 */
2947 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
2948 {
2949 struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
2950
2951 /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
2952 nescq->hw_cq.cq_number); */
2953 nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
2954
2955 if (nescq->ibcq.comp_handler)
2956 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
2957
2958 return;
2959 }
2960
2961
2962 /**
2963 * nes_manage_apbvt()
2964 */
2965 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
2966 u32 nic_index, u32 add_port)
2967 {
2968 struct nes_device *nesdev = nesvnic->nesdev;
2969 struct nes_hw_cqp_wqe *cqp_wqe;
2970 unsigned long flags;
2971 struct nes_cqp_request *cqp_request;
2972 int ret = 0;
2973 u16 major_code;
2974
2975 /* Send manage APBVT request to CQP */
2976 cqp_request = nes_get_cqp_request(nesdev);
2977 if (cqp_request == NULL) {
2978 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
2979 return -ENOMEM;
2980 }
2981 cqp_request->waiting = 1;
2982 cqp_wqe = &cqp_request->cqp_wqe;
2983
2984 nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
2985 (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
2986 accel_local_port, accel_local_port, nic_index);
2987
2988 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2989 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
2990 ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
2991 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2992 ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
2993
2994 nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
2995
2996 atomic_set(&cqp_request->refcount, 2);
2997 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
2998
2999 if (add_port == NES_MANAGE_APBVT_ADD)
3000 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3001 NES_EVENT_TIMEOUT);
3002 nes_debug(NES_DBG_QP, "Completed, ret=%u, CQP Major:Minor codes = 0x%04X:0x%04X\n",
3003 ret, cqp_request->major_code, cqp_request->minor_code);
3004 major_code = cqp_request->major_code;
3005 if (atomic_dec_and_test(&cqp_request->refcount)) {
3006 if (cqp_request->dynamic) {
3007 kfree(cqp_request);
3008 } else {
3009 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3010 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3011 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3012 }
3013 }
3014 if (!ret)
3015 return -ETIME;
3016 else if (major_code)
3017 return -EIO;
3018 else
3019 return 0;
3020 }
3021
3022
3023 /**
3024 * nes_manage_arp_cache
3025 */
3026 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3027 u32 ip_addr, u32 action)
3028 {
3029 struct nes_hw_cqp_wqe *cqp_wqe;
3030 struct nes_vnic *nesvnic = netdev_priv(netdev);
3031 struct nes_device *nesdev;
3032 struct nes_cqp_request *cqp_request;
3033 int arp_index;
3034
3035 nesdev = nesvnic->nesdev;
3036 arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3037 if (arp_index == -1) {
3038 return;
3039 }
3040
3041 /* update the ARP entry */
3042 cqp_request = nes_get_cqp_request(nesdev);
3043 if (cqp_request == NULL) {
3044 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3045 return;
3046 }
3047 cqp_request->waiting = 0;
3048 cqp_wqe = &cqp_request->cqp_wqe;
3049 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3050
3051 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3052 NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3053 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3054 (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3055 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3056
3057 if (action == NES_ARP_ADD) {
3058 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3059 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3060 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3061 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3062 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3063 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3064 } else {
3065 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3066 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3067 }
3068
3069 nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3070 nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3071
3072 atomic_set(&cqp_request->refcount, 1);
3073 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3074 }
3075
3076
3077 /**
3078 * flush_wqes
3079 */
3080 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3081 u32 which_wq, u32 wait_completion)
3082 {
3083 unsigned long flags;
3084 struct nes_cqp_request *cqp_request;
3085 struct nes_hw_cqp_wqe *cqp_wqe;
3086 int ret;
3087
3088 cqp_request = nes_get_cqp_request(nesdev);
3089 if (cqp_request == NULL) {
3090 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3091 return;
3092 }
3093 if (wait_completion) {
3094 cqp_request->waiting = 1;
3095 atomic_set(&cqp_request->refcount, 2);
3096 } else {
3097 cqp_request->waiting = 0;
3098 }
3099 cqp_wqe = &cqp_request->cqp_wqe;
3100 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3101
3102 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3103 cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3104 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3105
3106 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3107
3108 if (wait_completion) {
3109 /* Wait for CQP */
3110 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3111 NES_EVENT_TIMEOUT);
3112 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3113 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3114 ret, cqp_request->major_code, cqp_request->minor_code);
3115 if (atomic_dec_and_test(&cqp_request->refcount)) {
3116 if (cqp_request->dynamic) {
3117 kfree(cqp_request);
3118 } else {
3119 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3120 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3121 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3122 }
3123 }
3124 }
3125 }
Linux kernel - Deliverables
This page took 0.210233 seconds and 6 git commands to generate.