1 /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
13 /* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support
16 #include <linux/tcp.h>
18 #include <linux/ipv6.h>
19 #include <linux/crc32.h>
20 #include <linux/if_vlan.h>
21 #include <linux/jiffies.h>
22 #include <linux/phy.h>
24 #include <net/ip6_checksum.h>
26 #include "emac-sgmii.h"
28 /* EMAC base register offsets */
29 #define EMAC_MAC_CTRL 0x001480
30 #define EMAC_WOL_CTRL0 0x0014a0
31 #define EMAC_RSS_KEY0 0x0014b0
32 #define EMAC_H1TPD_BASE_ADDR_LO 0x0014e0
33 #define EMAC_H2TPD_BASE_ADDR_LO 0x0014e4
34 #define EMAC_H3TPD_BASE_ADDR_LO 0x0014e8
35 #define EMAC_INTER_SRAM_PART9 0x001534
36 #define EMAC_DESC_CTRL_0 0x001540
37 #define EMAC_DESC_CTRL_1 0x001544
38 #define EMAC_DESC_CTRL_2 0x001550
39 #define EMAC_DESC_CTRL_10 0x001554
40 #define EMAC_DESC_CTRL_12 0x001558
41 #define EMAC_DESC_CTRL_13 0x00155c
42 #define EMAC_DESC_CTRL_3 0x001560
43 #define EMAC_DESC_CTRL_4 0x001564
44 #define EMAC_DESC_CTRL_5 0x001568
45 #define EMAC_DESC_CTRL_14 0x00156c
46 #define EMAC_DESC_CTRL_15 0x001570
47 #define EMAC_DESC_CTRL_16 0x001574
48 #define EMAC_DESC_CTRL_6 0x001578
49 #define EMAC_DESC_CTRL_8 0x001580
50 #define EMAC_DESC_CTRL_9 0x001584
51 #define EMAC_DESC_CTRL_11 0x001588
52 #define EMAC_TXQ_CTRL_0 0x001590
53 #define EMAC_TXQ_CTRL_1 0x001594
54 #define EMAC_TXQ_CTRL_2 0x001598
55 #define EMAC_RXQ_CTRL_0 0x0015a0
56 #define EMAC_RXQ_CTRL_1 0x0015a4
57 #define EMAC_RXQ_CTRL_2 0x0015a8
58 #define EMAC_RXQ_CTRL_3 0x0015ac
59 #define EMAC_BASE_CPU_NUMBER 0x0015b8
60 #define EMAC_DMA_CTRL 0x0015c0
61 #define EMAC_MAILBOX_0 0x0015e0
62 #define EMAC_MAILBOX_5 0x0015e4
63 #define EMAC_MAILBOX_6 0x0015e8
64 #define EMAC_MAILBOX_13 0x0015ec
65 #define EMAC_MAILBOX_2 0x0015f4
66 #define EMAC_MAILBOX_3 0x0015f8
67 #define EMAC_MAILBOX_11 0x00160c
68 #define EMAC_AXI_MAST_CTRL 0x001610
69 #define EMAC_MAILBOX_12 0x001614
70 #define EMAC_MAILBOX_9 0x001618
71 #define EMAC_MAILBOX_10 0x00161c
72 #define EMAC_ATHR_HEADER_CTRL 0x001620
73 #define EMAC_CLK_GATE_CTRL 0x001814
74 #define EMAC_MISC_CTRL 0x001990
75 #define EMAC_MAILBOX_7 0x0019e0
76 #define EMAC_MAILBOX_8 0x0019e4
77 #define EMAC_MAILBOX_15 0x001bd4
78 #define EMAC_MAILBOX_16 0x001bd8
81 #define SINGLE_PAUSE_MODE 0x10000000
82 #define DEBUG_MODE 0x08000000
83 #define BROAD_EN 0x04000000
84 #define MULTI_ALL 0x02000000
85 #define RX_CHKSUM_EN 0x01000000
86 #define HUGE 0x00800000
87 #define SPEED(x) (((x) & 0x3) << 20)
88 #define SPEED_MASK SPEED(0x3)
89 #define SIMR 0x00080000
90 #define TPAUSE 0x00010000
91 #define PROM_MODE 0x00008000
92 #define VLAN_STRIP 0x00004000
93 #define PRLEN_BMSK 0x00003c00
95 #define HUGEN 0x00000200
96 #define FLCHK 0x00000100
97 #define PCRCE 0x00000080
98 #define CRCE 0x00000040
99 #define FULLD 0x00000020
100 #define MAC_LP_EN 0x00000010
101 #define RXFC 0x00000008
102 #define TXFC 0x00000004
103 #define RXEN 0x00000002
104 #define TXEN 0x00000001
108 #define LK_CHG_PME 0x20
109 #define LK_CHG_EN 0x10
110 #define MG_FRAME_PME 0x8
111 #define MG_FRAME_EN 0x4
112 #define WK_FRAME_EN 0x1
114 /* EMAC_DESC_CTRL_3 */
115 #define RFD_RING_SIZE_BMSK 0xfff
117 /* EMAC_DESC_CTRL_4 */
118 #define RX_BUFFER_SIZE_BMSK 0xffff
120 /* EMAC_DESC_CTRL_6 */
121 #define RRD_RING_SIZE_BMSK 0xfff
123 /* EMAC_DESC_CTRL_9 */
124 #define TPD_RING_SIZE_BMSK 0xffff
126 /* EMAC_TXQ_CTRL_0 */
127 #define NUM_TXF_BURST_PREF_BMSK 0xffff0000
128 #define NUM_TXF_BURST_PREF_SHFT 16
129 #define LS_8023_SP 0x80
130 #define TXQ_MODE 0x40
132 #define IP_OP_SP 0x10
133 #define NUM_TPD_BURST_PREF_BMSK 0xf
134 #define NUM_TPD_BURST_PREF_SHFT 0
136 /* EMAC_TXQ_CTRL_1 */
137 #define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK 0x7ff
139 /* EMAC_TXQ_CTRL_2 */
140 #define TXF_HWM_BMSK 0xfff0000
141 #define TXF_LWM_BMSK 0xfff
143 /* EMAC_RXQ_CTRL_0 */
144 #define RXQ_EN BIT(31)
145 #define CUT_THRU_EN BIT(30)
146 #define RSS_HASH_EN BIT(29)
147 #define NUM_RFD_BURST_PREF_BMSK 0x3f00000
148 #define NUM_RFD_BURST_PREF_SHFT 20
149 #define IDT_TABLE_SIZE_BMSK 0x1ff00
150 #define IDT_TABLE_SIZE_SHFT 8
153 /* EMAC_RXQ_CTRL_1 */
154 #define JUMBO_1KAH_BMSK 0xf000
155 #define JUMBO_1KAH_SHFT 12
156 #define RFD_PREF_LOW_TH 0x10
157 #define RFD_PREF_LOW_THRESHOLD_BMSK 0xfc0
158 #define RFD_PREF_LOW_THRESHOLD_SHFT 6
159 #define RFD_PREF_UP_TH 0x10
160 #define RFD_PREF_UP_THRESHOLD_BMSK 0x3f
161 #define RFD_PREF_UP_THRESHOLD_SHFT 0
163 /* EMAC_RXQ_CTRL_2 */
164 #define RXF_DOF_THRESFHOLD 0x1a0
165 #define RXF_DOF_THRESHOLD_BMSK 0xfff0000
166 #define RXF_DOF_THRESHOLD_SHFT 16
167 #define RXF_UOF_THRESFHOLD 0xbe
168 #define RXF_UOF_THRESHOLD_BMSK 0xfff
169 #define RXF_UOF_THRESHOLD_SHFT 0
171 /* EMAC_RXQ_CTRL_3 */
172 #define RXD_TIMER_BMSK 0xffff0000
173 #define RXD_THRESHOLD_BMSK 0xfff
174 #define RXD_THRESHOLD_SHFT 0
177 #define DMAW_DLY_CNT_BMSK 0xf0000
178 #define DMAW_DLY_CNT_SHFT 16
179 #define DMAR_DLY_CNT_BMSK 0xf800
180 #define DMAR_DLY_CNT_SHFT 11
181 #define DMAR_REQ_PRI 0x400
182 #define REGWRBLEN_BMSK 0x380
183 #define REGWRBLEN_SHFT 7
184 #define REGRDBLEN_BMSK 0x70
185 #define REGRDBLEN_SHFT 4
186 #define OUT_ORDER_MODE 0x4
187 #define ENH_ORDER_MODE 0x2
188 #define IN_ORDER_MODE 0x1
190 /* EMAC_MAILBOX_13 */
191 #define RFD3_PROC_IDX_BMSK 0xfff0000
192 #define RFD3_PROC_IDX_SHFT 16
193 #define RFD3_PROD_IDX_BMSK 0xfff
194 #define RFD3_PROD_IDX_SHFT 0
197 #define NTPD_CONS_IDX_BMSK 0xffff0000
198 #define NTPD_CONS_IDX_SHFT 16
201 #define RFD0_CONS_IDX_BMSK 0xfff
202 #define RFD0_CONS_IDX_SHFT 0
204 /* EMAC_MAILBOX_11 */
205 #define H3TPD_PROD_IDX_BMSK 0xffff0000
206 #define H3TPD_PROD_IDX_SHFT 16
208 /* EMAC_AXI_MAST_CTRL */
209 #define DATA_BYTE_SWAP 0x8
210 #define MAX_BOUND 0x2
211 #define MAX_BTYPE 0x1
213 /* EMAC_MAILBOX_12 */
214 #define H3TPD_CONS_IDX_BMSK 0xffff0000
215 #define H3TPD_CONS_IDX_SHFT 16
218 #define H2TPD_PROD_IDX_BMSK 0xffff
219 #define H2TPD_PROD_IDX_SHFT 0
221 /* EMAC_MAILBOX_10 */
222 #define H1TPD_CONS_IDX_BMSK 0xffff0000
223 #define H1TPD_CONS_IDX_SHFT 16
224 #define H2TPD_CONS_IDX_BMSK 0xffff
225 #define H2TPD_CONS_IDX_SHFT 0
227 /* EMAC_ATHR_HEADER_CTRL */
228 #define HEADER_CNT_EN 0x2
229 #define HEADER_ENABLE 0x1
232 #define RFD0_PROC_IDX_BMSK 0xfff0000
233 #define RFD0_PROC_IDX_SHFT 16
234 #define RFD0_PROD_IDX_BMSK 0xfff
235 #define RFD0_PROD_IDX_SHFT 0
238 #define RFD1_PROC_IDX_BMSK 0xfff0000
239 #define RFD1_PROC_IDX_SHFT 16
240 #define RFD1_PROD_IDX_BMSK 0xfff
241 #define RFD1_PROD_IDX_SHFT 0
244 #define RX_UNCPL_INT_EN 0x1
247 #define RFD2_CONS_IDX_BMSK 0xfff0000
248 #define RFD2_CONS_IDX_SHFT 16
249 #define RFD1_CONS_IDX_BMSK 0xfff
250 #define RFD1_CONS_IDX_SHFT 0
253 #define RFD3_CONS_IDX_BMSK 0xfff
254 #define RFD3_CONS_IDX_SHFT 0
256 /* EMAC_MAILBOX_15 */
257 #define NTPD_PROD_IDX_BMSK 0xffff
258 #define NTPD_PROD_IDX_SHFT 0
260 /* EMAC_MAILBOX_16 */
261 #define H1TPD_PROD_IDX_BMSK 0xffff
262 #define H1TPD_PROD_IDX_SHFT 0
264 #define RXQ0_RSS_HSTYP_IPV6_TCP_EN 0x20
265 #define RXQ0_RSS_HSTYP_IPV6_EN 0x10
266 #define RXQ0_RSS_HSTYP_IPV4_TCP_EN 0x8
267 #define RXQ0_RSS_HSTYP_IPV4_EN 0x4
269 /* EMAC_EMAC_WRAPPER_TX_TS_INX */
270 #define EMAC_WRAPPER_TX_TS_EMPTY BIT(31)
271 #define EMAC_WRAPPER_TX_TS_INX_BMSK 0xffff
275 unsigned long jiffies
;
278 #define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb)
279 #define EMAC_RSS_IDT_SIZE 256
280 #define JUMBO_1KAH 0x4
282 #define EMAC_TPD_LAST_FRAGMENT 0x80000000
283 #define EMAC_TPD_TSTAMP_SAVE 0x80000000
285 /* EMAC Errors in emac_rrd.word[3] */
286 #define EMAC_RRD_L4F BIT(14)
287 #define EMAC_RRD_IPF BIT(15)
288 #define EMAC_RRD_CRC BIT(21)
289 #define EMAC_RRD_FAE BIT(22)
290 #define EMAC_RRD_TRN BIT(23)
291 #define EMAC_RRD_RNT BIT(24)
292 #define EMAC_RRD_INC BIT(25)
293 #define EMAC_RRD_FOV BIT(29)
294 #define EMAC_RRD_LEN BIT(30)
296 /* Error bits that will result in a received frame being discarded */
297 #define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \
298 EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \
299 EMAC_RRD_FOV | EMAC_RRD_LEN)
300 #define EMAC_RRD_STATS_DW_IDX 3
302 #define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX)))
303 #define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX)))
304 #define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX)))
306 #define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)]))
307 #define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)]))
309 #define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD 8
311 #define ISR_RX_PKT (\
317 #define EMAC_MAC_IRQ_RES "core0"
319 void emac_mac_multicast_addr_set(struct emac_adapter
*adpt
, u8
*addr
)
321 u32 crc32
, bit
, reg
, mta
;
323 /* Calculate the CRC of the MAC address */
324 crc32
= ether_crc(ETH_ALEN
, addr
);
326 /* The HASH Table is an array of 2 32-bit registers. It is
327 * treated like an array of 64 bits (BitArray[hash_value]).
328 * Use the upper 6 bits of the above CRC as the hash value.
330 reg
= (crc32
>> 31) & 0x1;
331 bit
= (crc32
>> 26) & 0x1F;
333 mta
= readl(adpt
->base
+ EMAC_HASH_TAB_REG0
+ (reg
<< 2));
335 writel(mta
, adpt
->base
+ EMAC_HASH_TAB_REG0
+ (reg
<< 2));
338 void emac_mac_multicast_addr_clear(struct emac_adapter
*adpt
)
340 writel(0, adpt
->base
+ EMAC_HASH_TAB_REG0
);
341 writel(0, adpt
->base
+ EMAC_HASH_TAB_REG1
);
344 /* definitions for RSS */
345 #define EMAC_RSS_KEY(_i, _type) \
346 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))
347 #define EMAC_RSS_TBL(_i, _type) \
348 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))
350 /* Config MAC modes */
351 void emac_mac_mode_config(struct emac_adapter
*adpt
)
353 struct net_device
*netdev
= adpt
->netdev
;
356 mac
= readl(adpt
->base
+ EMAC_MAC_CTRL
);
357 mac
&= ~(VLAN_STRIP
| PROM_MODE
| MULTI_ALL
| MAC_LP_EN
);
359 if (netdev
->features
& NETIF_F_HW_VLAN_CTAG_RX
)
362 if (netdev
->flags
& IFF_PROMISC
)
365 if (netdev
->flags
& IFF_ALLMULTI
)
368 writel(mac
, adpt
->base
+ EMAC_MAC_CTRL
);
371 /* Config descriptor rings */
372 static void emac_mac_dma_rings_config(struct emac_adapter
*adpt
)
374 static const unsigned short tpd_q_offset
[] = {
375 EMAC_DESC_CTRL_8
, EMAC_H1TPD_BASE_ADDR_LO
,
376 EMAC_H2TPD_BASE_ADDR_LO
, EMAC_H3TPD_BASE_ADDR_LO
};
377 static const unsigned short rfd_q_offset
[] = {
378 EMAC_DESC_CTRL_2
, EMAC_DESC_CTRL_10
,
379 EMAC_DESC_CTRL_12
, EMAC_DESC_CTRL_13
};
380 static const unsigned short rrd_q_offset
[] = {
381 EMAC_DESC_CTRL_5
, EMAC_DESC_CTRL_14
,
382 EMAC_DESC_CTRL_15
, EMAC_DESC_CTRL_16
};
384 /* TPD (Transmit Packet Descriptor) */
385 writel(upper_32_bits(adpt
->tx_q
.tpd
.dma_addr
),
386 adpt
->base
+ EMAC_DESC_CTRL_1
);
388 writel(lower_32_bits(adpt
->tx_q
.tpd
.dma_addr
),
389 adpt
->base
+ tpd_q_offset
[0]);
391 writel(adpt
->tx_q
.tpd
.count
& TPD_RING_SIZE_BMSK
,
392 adpt
->base
+ EMAC_DESC_CTRL_9
);
394 /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */
395 writel(upper_32_bits(adpt
->rx_q
.rfd
.dma_addr
),
396 adpt
->base
+ EMAC_DESC_CTRL_0
);
398 writel(lower_32_bits(adpt
->rx_q
.rfd
.dma_addr
),
399 adpt
->base
+ rfd_q_offset
[0]);
400 writel(lower_32_bits(adpt
->rx_q
.rrd
.dma_addr
),
401 adpt
->base
+ rrd_q_offset
[0]);
403 writel(adpt
->rx_q
.rfd
.count
& RFD_RING_SIZE_BMSK
,
404 adpt
->base
+ EMAC_DESC_CTRL_3
);
405 writel(adpt
->rx_q
.rrd
.count
& RRD_RING_SIZE_BMSK
,
406 adpt
->base
+ EMAC_DESC_CTRL_6
);
408 writel(adpt
->rxbuf_size
& RX_BUFFER_SIZE_BMSK
,
409 adpt
->base
+ EMAC_DESC_CTRL_4
);
411 writel(0, adpt
->base
+ EMAC_DESC_CTRL_11
);
413 /* Load all of the base addresses above and ensure that triggering HW to
414 * read ring pointers is flushed
416 writel(1, adpt
->base
+ EMAC_INTER_SRAM_PART9
);
419 /* Config transmit parameters */
420 static void emac_mac_tx_config(struct emac_adapter
*adpt
)
424 writel((EMAC_MAX_TX_OFFLOAD_THRESH
>> 3) &
425 JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK
, adpt
->base
+ EMAC_TXQ_CTRL_1
);
427 val
= (adpt
->tpd_burst
<< NUM_TPD_BURST_PREF_SHFT
) &
428 NUM_TPD_BURST_PREF_BMSK
;
430 val
|= TXQ_MODE
| LS_8023_SP
;
431 val
|= (0x0100 << NUM_TXF_BURST_PREF_SHFT
) &
432 NUM_TXF_BURST_PREF_BMSK
;
434 writel(val
, adpt
->base
+ EMAC_TXQ_CTRL_0
);
435 emac_reg_update32(adpt
->base
+ EMAC_TXQ_CTRL_2
,
436 (TXF_HWM_BMSK
| TXF_LWM_BMSK
), 0);
439 /* Config receive parameters */
440 static void emac_mac_rx_config(struct emac_adapter
*adpt
)
444 val
= (adpt
->rfd_burst
<< NUM_RFD_BURST_PREF_SHFT
) &
445 NUM_RFD_BURST_PREF_BMSK
;
446 val
|= (SP_IPV6
| CUT_THRU_EN
);
448 writel(val
, adpt
->base
+ EMAC_RXQ_CTRL_0
);
450 val
= readl(adpt
->base
+ EMAC_RXQ_CTRL_1
);
451 val
&= ~(JUMBO_1KAH_BMSK
| RFD_PREF_LOW_THRESHOLD_BMSK
|
452 RFD_PREF_UP_THRESHOLD_BMSK
);
453 val
|= (JUMBO_1KAH
<< JUMBO_1KAH_SHFT
) |
454 (RFD_PREF_LOW_TH
<< RFD_PREF_LOW_THRESHOLD_SHFT
) |
455 (RFD_PREF_UP_TH
<< RFD_PREF_UP_THRESHOLD_SHFT
);
456 writel(val
, adpt
->base
+ EMAC_RXQ_CTRL_1
);
458 val
= readl(adpt
->base
+ EMAC_RXQ_CTRL_2
);
459 val
&= ~(RXF_DOF_THRESHOLD_BMSK
| RXF_UOF_THRESHOLD_BMSK
);
460 val
|= (RXF_DOF_THRESFHOLD
<< RXF_DOF_THRESHOLD_SHFT
) |
461 (RXF_UOF_THRESFHOLD
<< RXF_UOF_THRESHOLD_SHFT
);
462 writel(val
, adpt
->base
+ EMAC_RXQ_CTRL_2
);
464 val
= readl(adpt
->base
+ EMAC_RXQ_CTRL_3
);
465 val
&= ~(RXD_TIMER_BMSK
| RXD_THRESHOLD_BMSK
);
466 val
|= RXD_TH
<< RXD_THRESHOLD_SHFT
;
467 writel(val
, adpt
->base
+ EMAC_RXQ_CTRL_3
);
471 static void emac_mac_dma_config(struct emac_adapter
*adpt
)
473 u32 dma_ctrl
= DMAR_REQ_PRI
;
475 switch (adpt
->dma_order
) {
476 case emac_dma_ord_in
:
477 dma_ctrl
|= IN_ORDER_MODE
;
479 case emac_dma_ord_enh
:
480 dma_ctrl
|= ENH_ORDER_MODE
;
482 case emac_dma_ord_out
:
483 dma_ctrl
|= OUT_ORDER_MODE
;
489 dma_ctrl
|= (((u32
)adpt
->dmar_block
) << REGRDBLEN_SHFT
) &
491 dma_ctrl
|= (((u32
)adpt
->dmaw_block
) << REGWRBLEN_SHFT
) &
493 dma_ctrl
|= (((u32
)adpt
->dmar_dly_cnt
) << DMAR_DLY_CNT_SHFT
) &
495 dma_ctrl
|= (((u32
)adpt
->dmaw_dly_cnt
) << DMAW_DLY_CNT_SHFT
) &
498 /* config DMA and ensure that configuration is flushed to HW */
499 writel(dma_ctrl
, adpt
->base
+ EMAC_DMA_CTRL
);
502 /* set MAC address */
503 static void emac_set_mac_address(struct emac_adapter
*adpt
, u8
*addr
)
507 /* for example: 00-A0-C6-11-22-33
508 * 0<-->C6112233, 1<-->00A0.
512 sta
= (((u32
)addr
[2]) << 24) | (((u32
)addr
[3]) << 16) |
513 (((u32
)addr
[4]) << 8) | (((u32
)addr
[5]));
514 writel(sta
, adpt
->base
+ EMAC_MAC_STA_ADDR0
);
516 /* hight 32bit word */
517 sta
= (((u32
)addr
[0]) << 8) | (u32
)addr
[1];
518 writel(sta
, adpt
->base
+ EMAC_MAC_STA_ADDR1
);
521 static void emac_mac_config(struct emac_adapter
*adpt
)
523 struct net_device
*netdev
= adpt
->netdev
;
524 unsigned int max_frame
;
527 emac_set_mac_address(adpt
, netdev
->dev_addr
);
529 max_frame
= netdev
->mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+ VLAN_HLEN
;
530 adpt
->rxbuf_size
= netdev
->mtu
> EMAC_DEF_RX_BUF_SIZE
?
531 ALIGN(max_frame
, 8) : EMAC_DEF_RX_BUF_SIZE
;
533 emac_mac_dma_rings_config(adpt
);
535 writel(netdev
->mtu
+ ETH_HLEN
+ VLAN_HLEN
+ ETH_FCS_LEN
,
536 adpt
->base
+ EMAC_MAX_FRAM_LEN_CTRL
);
538 emac_mac_tx_config(adpt
);
539 emac_mac_rx_config(adpt
);
540 emac_mac_dma_config(adpt
);
542 val
= readl(adpt
->base
+ EMAC_AXI_MAST_CTRL
);
543 val
&= ~(DATA_BYTE_SWAP
| MAX_BOUND
);
545 writel(val
, adpt
->base
+ EMAC_AXI_MAST_CTRL
);
546 writel(0, adpt
->base
+ EMAC_CLK_GATE_CTRL
);
547 writel(RX_UNCPL_INT_EN
, adpt
->base
+ EMAC_MISC_CTRL
);
550 void emac_mac_reset(struct emac_adapter
*adpt
)
554 emac_reg_update32(adpt
->base
+ EMAC_DMA_MAS_CTRL
, 0, SOFT_RST
);
555 usleep_range(100, 150); /* reset may take up to 100usec */
557 /* interrupt clear-on-read */
558 emac_reg_update32(adpt
->base
+ EMAC_DMA_MAS_CTRL
, 0, INT_RD_CLR_EN
);
561 void emac_mac_start(struct emac_adapter
*adpt
)
563 struct phy_device
*phydev
= adpt
->phydev
;
566 /* enable tx queue */
567 emac_reg_update32(adpt
->base
+ EMAC_TXQ_CTRL_0
, 0, TXQ_EN
);
569 /* enable rx queue */
570 emac_reg_update32(adpt
->base
+ EMAC_RXQ_CTRL_0
, 0, RXQ_EN
);
572 /* enable mac control */
573 mac
= readl(adpt
->base
+ EMAC_MAC_CTRL
);
574 csr1
= readl(adpt
->csr
+ EMAC_EMAC_WRAPPER_CSR1
);
576 mac
|= TXEN
| RXEN
; /* enable RX/TX */
578 /* We don't have ethtool support yet, so force flow-control mode
583 /* setup link speed */
585 if (phydev
->speed
== SPEED_1000
) {
593 if (phydev
->duplex
== DUPLEX_FULL
)
598 /* other parameters */
599 mac
|= (CRCE
| PCRCE
);
600 mac
|= ((adpt
->preamble
<< PRLEN_SHFT
) & PRLEN_BMSK
);
603 mac
&= ~RX_CHKSUM_EN
;
604 mac
&= ~(HUGEN
| VLAN_STRIP
| TPAUSE
| SIMR
| HUGE
| MULTI_ALL
|
605 DEBUG_MODE
| SINGLE_PAUSE_MODE
);
607 writel_relaxed(csr1
, adpt
->csr
+ EMAC_EMAC_WRAPPER_CSR1
);
609 writel_relaxed(mac
, adpt
->base
+ EMAC_MAC_CTRL
);
611 /* enable interrupt read clear, low power sleep mode and
615 writel_relaxed(adpt
->irq_mod
, adpt
->base
+ EMAC_IRQ_MOD_TIM_INIT
);
616 writel_relaxed(INT_RD_CLR_EN
| LPW_MODE
| IRQ_MODERATOR_EN
|
617 IRQ_MODERATOR2_EN
, adpt
->base
+ EMAC_DMA_MAS_CTRL
);
619 emac_mac_mode_config(adpt
);
621 emac_reg_update32(adpt
->base
+ EMAC_ATHR_HEADER_CTRL
,
622 (HEADER_ENABLE
| HEADER_CNT_EN
), 0);
624 emac_reg_update32(adpt
->csr
+ EMAC_EMAC_WRAPPER_CSR2
, 0, WOL_EN
);
627 void emac_mac_stop(struct emac_adapter
*adpt
)
629 emac_reg_update32(adpt
->base
+ EMAC_RXQ_CTRL_0
, RXQ_EN
, 0);
630 emac_reg_update32(adpt
->base
+ EMAC_TXQ_CTRL_0
, TXQ_EN
, 0);
631 emac_reg_update32(adpt
->base
+ EMAC_MAC_CTRL
, TXEN
| RXEN
, 0);
632 usleep_range(1000, 1050); /* stopping mac may take upto 1msec */
635 /* Free all descriptors of given transmit queue */
636 static void emac_tx_q_descs_free(struct emac_adapter
*adpt
)
638 struct emac_tx_queue
*tx_q
= &adpt
->tx_q
;
642 /* ring already cleared, nothing to do */
643 if (!tx_q
->tpd
.tpbuff
)
646 for (i
= 0; i
< tx_q
->tpd
.count
; i
++) {
647 struct emac_buffer
*tpbuf
= GET_TPD_BUFFER(tx_q
, i
);
649 if (tpbuf
->dma_addr
) {
650 dma_unmap_single(adpt
->netdev
->dev
.parent
,
651 tpbuf
->dma_addr
, tpbuf
->length
,
656 dev_kfree_skb_any(tpbuf
->skb
);
661 size
= sizeof(struct emac_buffer
) * tx_q
->tpd
.count
;
662 memset(tx_q
->tpd
.tpbuff
, 0, size
);
664 /* clear the descriptor ring */
665 memset(tx_q
->tpd
.v_addr
, 0, tx_q
->tpd
.size
);
667 tx_q
->tpd
.consume_idx
= 0;
668 tx_q
->tpd
.produce_idx
= 0;
671 /* Free all descriptors of given receive queue */
672 static void emac_rx_q_free_descs(struct emac_adapter
*adpt
)
674 struct device
*dev
= adpt
->netdev
->dev
.parent
;
675 struct emac_rx_queue
*rx_q
= &adpt
->rx_q
;
679 /* ring already cleared, nothing to do */
680 if (!rx_q
->rfd
.rfbuff
)
683 for (i
= 0; i
< rx_q
->rfd
.count
; i
++) {
684 struct emac_buffer
*rfbuf
= GET_RFD_BUFFER(rx_q
, i
);
686 if (rfbuf
->dma_addr
) {
687 dma_unmap_single(dev
, rfbuf
->dma_addr
, rfbuf
->length
,
692 dev_kfree_skb(rfbuf
->skb
);
697 size
= sizeof(struct emac_buffer
) * rx_q
->rfd
.count
;
698 memset(rx_q
->rfd
.rfbuff
, 0, size
);
700 /* clear the descriptor rings */
701 memset(rx_q
->rrd
.v_addr
, 0, rx_q
->rrd
.size
);
702 rx_q
->rrd
.produce_idx
= 0;
703 rx_q
->rrd
.consume_idx
= 0;
705 memset(rx_q
->rfd
.v_addr
, 0, rx_q
->rfd
.size
);
706 rx_q
->rfd
.produce_idx
= 0;
707 rx_q
->rfd
.consume_idx
= 0;
710 /* Free all buffers associated with given transmit queue */
711 static void emac_tx_q_bufs_free(struct emac_adapter
*adpt
)
713 struct emac_tx_queue
*tx_q
= &adpt
->tx_q
;
715 emac_tx_q_descs_free(adpt
);
717 kfree(tx_q
->tpd
.tpbuff
);
718 tx_q
->tpd
.tpbuff
= NULL
;
719 tx_q
->tpd
.v_addr
= NULL
;
720 tx_q
->tpd
.dma_addr
= 0;
724 /* Allocate TX descriptor ring for the given transmit queue */
725 static int emac_tx_q_desc_alloc(struct emac_adapter
*adpt
,
726 struct emac_tx_queue
*tx_q
)
728 struct emac_ring_header
*ring_header
= &adpt
->ring_header
;
731 size
= sizeof(struct emac_buffer
) * tx_q
->tpd
.count
;
732 tx_q
->tpd
.tpbuff
= kzalloc(size
, GFP_KERNEL
);
733 if (!tx_q
->tpd
.tpbuff
)
736 tx_q
->tpd
.size
= tx_q
->tpd
.count
* (adpt
->tpd_size
* 4);
737 tx_q
->tpd
.dma_addr
= ring_header
->dma_addr
+ ring_header
->used
;
738 tx_q
->tpd
.v_addr
= ring_header
->v_addr
+ ring_header
->used
;
739 ring_header
->used
+= ALIGN(tx_q
->tpd
.size
, 8);
740 tx_q
->tpd
.produce_idx
= 0;
741 tx_q
->tpd
.consume_idx
= 0;
746 /* Free all buffers associated with given transmit queue */
747 static void emac_rx_q_bufs_free(struct emac_adapter
*adpt
)
749 struct emac_rx_queue
*rx_q
= &adpt
->rx_q
;
751 emac_rx_q_free_descs(adpt
);
753 kfree(rx_q
->rfd
.rfbuff
);
754 rx_q
->rfd
.rfbuff
= NULL
;
756 rx_q
->rfd
.v_addr
= NULL
;
757 rx_q
->rfd
.dma_addr
= 0;
760 rx_q
->rrd
.v_addr
= NULL
;
761 rx_q
->rrd
.dma_addr
= 0;
765 /* Allocate RX descriptor rings for the given receive queue */
766 static int emac_rx_descs_alloc(struct emac_adapter
*adpt
)
768 struct emac_ring_header
*ring_header
= &adpt
->ring_header
;
769 struct emac_rx_queue
*rx_q
= &adpt
->rx_q
;
772 size
= sizeof(struct emac_buffer
) * rx_q
->rfd
.count
;
773 rx_q
->rfd
.rfbuff
= kzalloc(size
, GFP_KERNEL
);
774 if (!rx_q
->rfd
.rfbuff
)
777 rx_q
->rrd
.size
= rx_q
->rrd
.count
* (adpt
->rrd_size
* 4);
778 rx_q
->rfd
.size
= rx_q
->rfd
.count
* (adpt
->rfd_size
* 4);
780 rx_q
->rrd
.dma_addr
= ring_header
->dma_addr
+ ring_header
->used
;
781 rx_q
->rrd
.v_addr
= ring_header
->v_addr
+ ring_header
->used
;
782 ring_header
->used
+= ALIGN(rx_q
->rrd
.size
, 8);
784 rx_q
->rfd
.dma_addr
= ring_header
->dma_addr
+ ring_header
->used
;
785 rx_q
->rfd
.v_addr
= ring_header
->v_addr
+ ring_header
->used
;
786 ring_header
->used
+= ALIGN(rx_q
->rfd
.size
, 8);
788 rx_q
->rrd
.produce_idx
= 0;
789 rx_q
->rrd
.consume_idx
= 0;
791 rx_q
->rfd
.produce_idx
= 0;
792 rx_q
->rfd
.consume_idx
= 0;
797 /* Allocate all TX and RX descriptor rings */
798 int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter
*adpt
)
800 struct emac_ring_header
*ring_header
= &adpt
->ring_header
;
801 struct device
*dev
= adpt
->netdev
->dev
.parent
;
802 unsigned int num_tx_descs
= adpt
->tx_desc_cnt
;
803 unsigned int num_rx_descs
= adpt
->rx_desc_cnt
;
806 adpt
->tx_q
.tpd
.count
= adpt
->tx_desc_cnt
;
808 adpt
->rx_q
.rrd
.count
= adpt
->rx_desc_cnt
;
809 adpt
->rx_q
.rfd
.count
= adpt
->rx_desc_cnt
;
811 /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,
812 * hence the additional padding bytes are allocated.
814 ring_header
->size
= num_tx_descs
* (adpt
->tpd_size
* 4) +
815 num_rx_descs
* (adpt
->rfd_size
* 4) +
816 num_rx_descs
* (adpt
->rrd_size
* 4) +
817 8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */
819 ring_header
->used
= 0;
820 ring_header
->v_addr
= dma_zalloc_coherent(dev
, ring_header
->size
,
821 &ring_header
->dma_addr
,
823 if (!ring_header
->v_addr
)
826 ring_header
->used
= ALIGN(ring_header
->dma_addr
, 8) -
827 ring_header
->dma_addr
;
829 ret
= emac_tx_q_desc_alloc(adpt
, &adpt
->tx_q
);
831 netdev_err(adpt
->netdev
, "error: Tx Queue alloc failed\n");
835 ret
= emac_rx_descs_alloc(adpt
);
837 netdev_err(adpt
->netdev
, "error: Rx Queue alloc failed\n");
844 emac_tx_q_bufs_free(adpt
);
846 dma_free_coherent(dev
, ring_header
->size
,
847 ring_header
->v_addr
, ring_header
->dma_addr
);
849 ring_header
->v_addr
= NULL
;
850 ring_header
->dma_addr
= 0;
851 ring_header
->size
= 0;
852 ring_header
->used
= 0;
857 /* Free all TX and RX descriptor rings */
858 void emac_mac_rx_tx_rings_free_all(struct emac_adapter
*adpt
)
860 struct emac_ring_header
*ring_header
= &adpt
->ring_header
;
861 struct device
*dev
= adpt
->netdev
->dev
.parent
;
863 emac_tx_q_bufs_free(adpt
);
864 emac_rx_q_bufs_free(adpt
);
866 dma_free_coherent(dev
, ring_header
->size
,
867 ring_header
->v_addr
, ring_header
->dma_addr
);
869 ring_header
->v_addr
= NULL
;
870 ring_header
->dma_addr
= 0;
871 ring_header
->size
= 0;
872 ring_header
->used
= 0;
875 /* Initialize descriptor rings */
876 static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter
*adpt
)
880 adpt
->tx_q
.tpd
.produce_idx
= 0;
881 adpt
->tx_q
.tpd
.consume_idx
= 0;
882 for (i
= 0; i
< adpt
->tx_q
.tpd
.count
; i
++)
883 adpt
->tx_q
.tpd
.tpbuff
[i
].dma_addr
= 0;
885 adpt
->rx_q
.rrd
.produce_idx
= 0;
886 adpt
->rx_q
.rrd
.consume_idx
= 0;
887 adpt
->rx_q
.rfd
.produce_idx
= 0;
888 adpt
->rx_q
.rfd
.consume_idx
= 0;
889 for (i
= 0; i
< adpt
->rx_q
.rfd
.count
; i
++)
890 adpt
->rx_q
.rfd
.rfbuff
[i
].dma_addr
= 0;
893 /* Produce new receive free descriptor */
894 static void emac_mac_rx_rfd_create(struct emac_adapter
*adpt
,
895 struct emac_rx_queue
*rx_q
,
898 u32
*hw_rfd
= EMAC_RFD(rx_q
, adpt
->rfd_size
, rx_q
->rfd
.produce_idx
);
900 *(hw_rfd
++) = lower_32_bits(addr
);
901 *hw_rfd
= upper_32_bits(addr
);
903 if (++rx_q
->rfd
.produce_idx
== rx_q
->rfd
.count
)
904 rx_q
->rfd
.produce_idx
= 0;
907 /* Fill up receive queue's RFD with preallocated receive buffers */
908 static void emac_mac_rx_descs_refill(struct emac_adapter
*adpt
,
909 struct emac_rx_queue
*rx_q
)
911 struct emac_buffer
*curr_rxbuf
;
912 struct emac_buffer
*next_rxbuf
;
913 unsigned int count
= 0;
914 u32 next_produce_idx
;
916 next_produce_idx
= rx_q
->rfd
.produce_idx
+ 1;
917 if (next_produce_idx
== rx_q
->rfd
.count
)
918 next_produce_idx
= 0;
920 curr_rxbuf
= GET_RFD_BUFFER(rx_q
, rx_q
->rfd
.produce_idx
);
921 next_rxbuf
= GET_RFD_BUFFER(rx_q
, next_produce_idx
);
923 /* this always has a blank rx_buffer*/
924 while (!next_rxbuf
->dma_addr
) {
928 skb
= netdev_alloc_skb_ip_align(adpt
->netdev
, adpt
->rxbuf_size
);
932 curr_rxbuf
->dma_addr
=
933 dma_map_single(adpt
->netdev
->dev
.parent
, skb
->data
,
934 curr_rxbuf
->length
, DMA_FROM_DEVICE
);
935 ret
= dma_mapping_error(adpt
->netdev
->dev
.parent
,
936 curr_rxbuf
->dma_addr
);
941 curr_rxbuf
->skb
= skb
;
942 curr_rxbuf
->length
= adpt
->rxbuf_size
;
944 emac_mac_rx_rfd_create(adpt
, rx_q
, curr_rxbuf
->dma_addr
);
945 next_produce_idx
= rx_q
->rfd
.produce_idx
+ 1;
946 if (next_produce_idx
== rx_q
->rfd
.count
)
947 next_produce_idx
= 0;
949 curr_rxbuf
= GET_RFD_BUFFER(rx_q
, rx_q
->rfd
.produce_idx
);
950 next_rxbuf
= GET_RFD_BUFFER(rx_q
, next_produce_idx
);
955 u32 prod_idx
= (rx_q
->rfd
.produce_idx
<< rx_q
->produce_shift
) &
957 emac_reg_update32(adpt
->base
+ rx_q
->produce_reg
,
958 rx_q
->produce_mask
, prod_idx
);
962 static void emac_adjust_link(struct net_device
*netdev
)
964 struct emac_adapter
*adpt
= netdev_priv(netdev
);
965 struct phy_device
*phydev
= netdev
->phydev
;
968 emac_mac_start(adpt
);
972 phy_print_status(phydev
);
975 /* Bringup the interface/HW */
976 int emac_mac_up(struct emac_adapter
*adpt
)
978 struct net_device
*netdev
= adpt
->netdev
;
979 struct emac_irq
*irq
= &adpt
->irq
;
982 emac_mac_rx_tx_ring_reset_all(adpt
);
983 emac_mac_config(adpt
);
985 ret
= request_irq(irq
->irq
, emac_isr
, 0, EMAC_MAC_IRQ_RES
, irq
);
987 netdev_err(adpt
->netdev
, "could not request %s irq\n",
992 emac_mac_rx_descs_refill(adpt
, &adpt
->rx_q
);
994 ret
= phy_connect_direct(netdev
, adpt
->phydev
, emac_adjust_link
,
995 PHY_INTERFACE_MODE_SGMII
);
997 netdev_err(adpt
->netdev
, "could not connect phy\n");
998 free_irq(irq
->irq
, irq
);
1002 /* enable mac irq */
1003 writel((u32
)~DIS_INT
, adpt
->base
+ EMAC_INT_STATUS
);
1004 writel(adpt
->irq
.mask
, adpt
->base
+ EMAC_INT_MASK
);
1006 adpt
->phydev
->irq
= PHY_IGNORE_INTERRUPT
;
1007 phy_start(adpt
->phydev
);
1009 napi_enable(&adpt
->rx_q
.napi
);
1010 netif_start_queue(netdev
);
1015 /* Bring down the interface/HW */
1016 void emac_mac_down(struct emac_adapter
*adpt
)
1018 struct net_device
*netdev
= adpt
->netdev
;
1020 netif_stop_queue(netdev
);
1021 napi_disable(&adpt
->rx_q
.napi
);
1023 phy_stop(adpt
->phydev
);
1024 phy_disconnect(adpt
->phydev
);
1026 /* disable mac irq */
1027 writel(DIS_INT
, adpt
->base
+ EMAC_INT_STATUS
);
1028 writel(0, adpt
->base
+ EMAC_INT_MASK
);
1029 synchronize_irq(adpt
->irq
.irq
);
1030 free_irq(adpt
->irq
.irq
, &adpt
->irq
);
1032 emac_mac_reset(adpt
);
1034 emac_tx_q_descs_free(adpt
);
1035 netdev_reset_queue(adpt
->netdev
);
1036 emac_rx_q_free_descs(adpt
);
1039 /* Consume next received packet descriptor */
1040 static bool emac_rx_process_rrd(struct emac_adapter
*adpt
,
1041 struct emac_rx_queue
*rx_q
,
1042 struct emac_rrd
*rrd
)
1044 u32
*hw_rrd
= EMAC_RRD(rx_q
, adpt
->rrd_size
, rx_q
->rrd
.consume_idx
);
1046 rrd
->word
[3] = *(hw_rrd
+ 3);
1054 rrd
->word
[0] = *(hw_rrd
++);
1055 rrd
->word
[1] = *(hw_rrd
++);
1056 rrd
->word
[2] = *(hw_rrd
++);
1058 if (unlikely(RRD_NOR(rrd
) != 1)) {
1059 netdev_err(adpt
->netdev
,
1060 "error: multi-RFD not support yet! nor:%lu\n",
1064 /* mark rrd as processed */
1065 RRD_UPDT_SET(rrd
, 0);
1066 *hw_rrd
= rrd
->word
[3];
1068 if (++rx_q
->rrd
.consume_idx
== rx_q
->rrd
.count
)
1069 rx_q
->rrd
.consume_idx
= 0;
1074 /* Produce new transmit descriptor */
1075 static void emac_tx_tpd_create(struct emac_adapter
*adpt
,
1076 struct emac_tx_queue
*tx_q
, struct emac_tpd
*tpd
)
1080 tx_q
->tpd
.last_produce_idx
= tx_q
->tpd
.produce_idx
;
1081 hw_tpd
= EMAC_TPD(tx_q
, adpt
->tpd_size
, tx_q
->tpd
.produce_idx
);
1083 if (++tx_q
->tpd
.produce_idx
== tx_q
->tpd
.count
)
1084 tx_q
->tpd
.produce_idx
= 0;
1086 *(hw_tpd
++) = tpd
->word
[0];
1087 *(hw_tpd
++) = tpd
->word
[1];
1088 *(hw_tpd
++) = tpd
->word
[2];
1089 *hw_tpd
= tpd
->word
[3];
1092 /* Mark the last transmit descriptor as such (for the transmit packet) */
1093 static void emac_tx_tpd_mark_last(struct emac_adapter
*adpt
,
1094 struct emac_tx_queue
*tx_q
)
1097 EMAC_TPD(tx_q
, adpt
->tpd_size
, tx_q
->tpd
.last_produce_idx
);
1100 tmp_tpd
= *(hw_tpd
+ 1);
1101 tmp_tpd
|= EMAC_TPD_LAST_FRAGMENT
;
1102 *(hw_tpd
+ 1) = tmp_tpd
;
1105 static void emac_rx_rfd_clean(struct emac_rx_queue
*rx_q
, struct emac_rrd
*rrd
)
1107 struct emac_buffer
*rfbuf
= rx_q
->rfd
.rfbuff
;
1108 u32 consume_idx
= RRD_SI(rrd
);
1111 for (i
= 0; i
< RRD_NOR(rrd
); i
++) {
1112 rfbuf
[consume_idx
].skb
= NULL
;
1113 if (++consume_idx
== rx_q
->rfd
.count
)
1117 rx_q
->rfd
.consume_idx
= consume_idx
;
1118 rx_q
->rfd
.process_idx
= consume_idx
;
1121 /* Push the received skb to upper layers */
1122 static void emac_receive_skb(struct emac_rx_queue
*rx_q
,
1123 struct sk_buff
*skb
,
1124 u16 vlan_tag
, bool vlan_flag
)
1129 EMAC_TAG_TO_VLAN(vlan_tag
, vlan
);
1130 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), vlan
);
1133 napi_gro_receive(&rx_q
->napi
, skb
);
1136 /* Process receive event */
1137 void emac_mac_rx_process(struct emac_adapter
*adpt
, struct emac_rx_queue
*rx_q
,
1138 int *num_pkts
, int max_pkts
)
1140 u32 proc_idx
, hw_consume_idx
, num_consume_pkts
;
1141 struct net_device
*netdev
= adpt
->netdev
;
1142 struct emac_buffer
*rfbuf
;
1143 unsigned int count
= 0;
1144 struct emac_rrd rrd
;
1145 struct sk_buff
*skb
;
1148 reg
= readl_relaxed(adpt
->base
+ rx_q
->consume_reg
);
1150 hw_consume_idx
= (reg
& rx_q
->consume_mask
) >> rx_q
->consume_shift
;
1151 num_consume_pkts
= (hw_consume_idx
>= rx_q
->rrd
.consume_idx
) ?
1152 (hw_consume_idx
- rx_q
->rrd
.consume_idx
) :
1153 (hw_consume_idx
+ rx_q
->rrd
.count
- rx_q
->rrd
.consume_idx
);
1156 if (!num_consume_pkts
)
1159 if (!emac_rx_process_rrd(adpt
, rx_q
, &rrd
))
1162 if (likely(RRD_NOR(&rrd
) == 1)) {
1164 rfbuf
= GET_RFD_BUFFER(rx_q
, RRD_SI(&rrd
));
1165 dma_unmap_single(adpt
->netdev
->dev
.parent
,
1166 rfbuf
->dma_addr
, rfbuf
->length
,
1168 rfbuf
->dma_addr
= 0;
1171 netdev_err(adpt
->netdev
,
1172 "error: multi-RFD not support yet!\n");
1175 emac_rx_rfd_clean(rx_q
, &rrd
);
1179 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags
1180 * with DF set are marked as error), drop packets based on the
1181 * error mask rather than the summary bit (ignoring L4F errors)
1183 if (rrd
.word
[EMAC_RRD_STATS_DW_IDX
] & EMAC_RRD_ERROR
) {
1184 netif_dbg(adpt
, rx_status
, adpt
->netdev
,
1185 "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
1186 rrd
.word
[0], rrd
.word
[1],
1187 rrd
.word
[2], rrd
.word
[3]);
1193 skb_put(skb
, RRD_PKT_SIZE(&rrd
) - ETH_FCS_LEN
);
1195 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1196 if (netdev
->features
& NETIF_F_RXCSUM
)
1197 skb
->ip_summed
= RRD_L4F(&rrd
) ?
1198 CHECKSUM_NONE
: CHECKSUM_UNNECESSARY
;
1200 skb_checksum_none_assert(skb
);
1202 emac_receive_skb(rx_q
, skb
, (u16
)RRD_CVALN_TAG(&rrd
),
1203 (bool)RRD_CVTAG(&rrd
));
1205 netdev
->last_rx
= jiffies
;
1207 } while (*num_pkts
< max_pkts
);
1210 proc_idx
= (rx_q
->rfd
.process_idx
<< rx_q
->process_shft
) &
1212 emac_reg_update32(adpt
->base
+ rx_q
->process_reg
,
1213 rx_q
->process_mask
, proc_idx
);
1214 emac_mac_rx_descs_refill(adpt
, rx_q
);
1218 /* get the number of free transmit descriptors */
1219 static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue
*tx_q
)
1221 u32 produce_idx
= tx_q
->tpd
.produce_idx
;
1222 u32 consume_idx
= tx_q
->tpd
.consume_idx
;
1224 return (consume_idx
> produce_idx
) ?
1225 (consume_idx
- produce_idx
- 1) :
1226 (tx_q
->tpd
.count
+ consume_idx
- produce_idx
- 1);
1229 /* Process transmit event */
1230 void emac_mac_tx_process(struct emac_adapter
*adpt
, struct emac_tx_queue
*tx_q
)
1232 u32 reg
= readl_relaxed(adpt
->base
+ tx_q
->consume_reg
);
1233 u32 hw_consume_idx
, pkts_compl
= 0, bytes_compl
= 0;
1234 struct emac_buffer
*tpbuf
;
1236 hw_consume_idx
= (reg
& tx_q
->consume_mask
) >> tx_q
->consume_shift
;
1238 while (tx_q
->tpd
.consume_idx
!= hw_consume_idx
) {
1239 tpbuf
= GET_TPD_BUFFER(tx_q
, tx_q
->tpd
.consume_idx
);
1240 if (tpbuf
->dma_addr
) {
1241 dma_unmap_single(adpt
->netdev
->dev
.parent
,
1242 tpbuf
->dma_addr
, tpbuf
->length
,
1244 tpbuf
->dma_addr
= 0;
1249 bytes_compl
+= tpbuf
->skb
->len
;
1250 dev_kfree_skb_irq(tpbuf
->skb
);
1254 if (++tx_q
->tpd
.consume_idx
== tx_q
->tpd
.count
)
1255 tx_q
->tpd
.consume_idx
= 0;
1258 netdev_completed_queue(adpt
->netdev
, pkts_compl
, bytes_compl
);
1260 if (netif_queue_stopped(adpt
->netdev
))
1261 if (emac_tpd_num_free_descs(tx_q
) > (MAX_SKB_FRAGS
+ 1))
1262 netif_wake_queue(adpt
->netdev
);
1265 /* Initialize all queue data structures */
1266 void emac_mac_rx_tx_ring_init_all(struct platform_device
*pdev
,
1267 struct emac_adapter
*adpt
)
1269 adpt
->rx_q
.netdev
= adpt
->netdev
;
1271 adpt
->rx_q
.produce_reg
= EMAC_MAILBOX_0
;
1272 adpt
->rx_q
.produce_mask
= RFD0_PROD_IDX_BMSK
;
1273 adpt
->rx_q
.produce_shift
= RFD0_PROD_IDX_SHFT
;
1275 adpt
->rx_q
.process_reg
= EMAC_MAILBOX_0
;
1276 adpt
->rx_q
.process_mask
= RFD0_PROC_IDX_BMSK
;
1277 adpt
->rx_q
.process_shft
= RFD0_PROC_IDX_SHFT
;
1279 adpt
->rx_q
.consume_reg
= EMAC_MAILBOX_3
;
1280 adpt
->rx_q
.consume_mask
= RFD0_CONS_IDX_BMSK
;
1281 adpt
->rx_q
.consume_shift
= RFD0_CONS_IDX_SHFT
;
1283 adpt
->rx_q
.irq
= &adpt
->irq
;
1284 adpt
->rx_q
.intr
= adpt
->irq
.mask
& ISR_RX_PKT
;
1286 adpt
->tx_q
.produce_reg
= EMAC_MAILBOX_15
;
1287 adpt
->tx_q
.produce_mask
= NTPD_PROD_IDX_BMSK
;
1288 adpt
->tx_q
.produce_shift
= NTPD_PROD_IDX_SHFT
;
1290 adpt
->tx_q
.consume_reg
= EMAC_MAILBOX_2
;
1291 adpt
->tx_q
.consume_mask
= NTPD_CONS_IDX_BMSK
;
1292 adpt
->tx_q
.consume_shift
= NTPD_CONS_IDX_SHFT
;
1295 /* Fill up transmit descriptors with TSO and Checksum offload information */
1296 static int emac_tso_csum(struct emac_adapter
*adpt
,
1297 struct emac_tx_queue
*tx_q
,
1298 struct sk_buff
*skb
,
1299 struct emac_tpd
*tpd
)
1301 unsigned int hdr_len
;
1304 if (skb_is_gso(skb
)) {
1305 if (skb_header_cloned(skb
)) {
1306 ret
= pskb_expand_head(skb
, 0, 0, GFP_ATOMIC
);
1311 if (skb
->protocol
== htons(ETH_P_IP
)) {
1312 u32 pkt_len
= ((unsigned char *)ip_hdr(skb
) - skb
->data
)
1313 + ntohs(ip_hdr(skb
)->tot_len
);
1314 if (skb
->len
> pkt_len
)
1315 pskb_trim(skb
, pkt_len
);
1318 hdr_len
= skb_transport_offset(skb
) + tcp_hdrlen(skb
);
1319 if (unlikely(skb
->len
== hdr_len
)) {
1320 /* we only need to do csum */
1321 netif_warn(adpt
, tx_err
, adpt
->netdev
,
1322 "tso not needed for packet with 0 data\n");
1326 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
) {
1327 ip_hdr(skb
)->check
= 0;
1328 tcp_hdr(skb
)->check
=
1329 ~csum_tcpudp_magic(ip_hdr(skb
)->saddr
,
1332 TPD_IPV4_SET(tpd
, 1);
1335 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
) {
1336 /* ipv6 tso need an extra tpd */
1337 struct emac_tpd extra_tpd
;
1339 memset(tpd
, 0, sizeof(*tpd
));
1340 memset(&extra_tpd
, 0, sizeof(extra_tpd
));
1342 ipv6_hdr(skb
)->payload_len
= 0;
1343 tcp_hdr(skb
)->check
=
1344 ~csum_ipv6_magic(&ipv6_hdr(skb
)->saddr
,
1345 &ipv6_hdr(skb
)->daddr
,
1347 TPD_PKT_LEN_SET(&extra_tpd
, skb
->len
);
1348 TPD_LSO_SET(&extra_tpd
, 1);
1349 TPD_LSOV_SET(&extra_tpd
, 1);
1350 emac_tx_tpd_create(adpt
, tx_q
, &extra_tpd
);
1351 TPD_LSOV_SET(tpd
, 1);
1354 TPD_LSO_SET(tpd
, 1);
1355 TPD_TCPHDR_OFFSET_SET(tpd
, skb_transport_offset(skb
));
1356 TPD_MSS_SET(tpd
, skb_shinfo(skb
)->gso_size
);
1361 if (likely(skb
->ip_summed
== CHECKSUM_PARTIAL
)) {
1362 unsigned int css
, cso
;
1364 cso
= skb_transport_offset(skb
);
1365 if (unlikely(cso
& 0x1)) {
1366 netdev_err(adpt
->netdev
,
1367 "error: payload offset should be even\n");
1370 css
= cso
+ skb
->csum_offset
;
1372 TPD_PAYLOAD_OFFSET_SET(tpd
, cso
>> 1);
1373 TPD_CXSUM_OFFSET_SET(tpd
, css
>> 1);
1374 TPD_CSX_SET(tpd
, 1);
1380 /* Fill up transmit descriptors */
1381 static void emac_tx_fill_tpd(struct emac_adapter
*adpt
,
1382 struct emac_tx_queue
*tx_q
, struct sk_buff
*skb
,
1383 struct emac_tpd
*tpd
)
1385 unsigned int nr_frags
= skb_shinfo(skb
)->nr_frags
;
1386 unsigned int first
= tx_q
->tpd
.produce_idx
;
1387 unsigned int len
= skb_headlen(skb
);
1388 struct emac_buffer
*tpbuf
= NULL
;
1389 unsigned int mapped_len
= 0;
1394 /* if Large Segment Offload is (in TCP Segmentation Offload struct) */
1396 mapped_len
= skb_transport_offset(skb
) + tcp_hdrlen(skb
);
1398 tpbuf
= GET_TPD_BUFFER(tx_q
, tx_q
->tpd
.produce_idx
);
1399 tpbuf
->length
= mapped_len
;
1400 tpbuf
->dma_addr
= dma_map_single(adpt
->netdev
->dev
.parent
,
1401 skb
->data
, tpbuf
->length
,
1403 ret
= dma_mapping_error(adpt
->netdev
->dev
.parent
,
1408 TPD_BUFFER_ADDR_L_SET(tpd
, lower_32_bits(tpbuf
->dma_addr
));
1409 TPD_BUFFER_ADDR_H_SET(tpd
, upper_32_bits(tpbuf
->dma_addr
));
1410 TPD_BUF_LEN_SET(tpd
, tpbuf
->length
);
1411 emac_tx_tpd_create(adpt
, tx_q
, tpd
);
1415 if (mapped_len
< len
) {
1416 tpbuf
= GET_TPD_BUFFER(tx_q
, tx_q
->tpd
.produce_idx
);
1417 tpbuf
->length
= len
- mapped_len
;
1418 tpbuf
->dma_addr
= dma_map_single(adpt
->netdev
->dev
.parent
,
1419 skb
->data
+ mapped_len
,
1420 tpbuf
->length
, DMA_TO_DEVICE
);
1421 ret
= dma_mapping_error(adpt
->netdev
->dev
.parent
,
1426 TPD_BUFFER_ADDR_L_SET(tpd
, lower_32_bits(tpbuf
->dma_addr
));
1427 TPD_BUFFER_ADDR_H_SET(tpd
, upper_32_bits(tpbuf
->dma_addr
));
1428 TPD_BUF_LEN_SET(tpd
, tpbuf
->length
);
1429 emac_tx_tpd_create(adpt
, tx_q
, tpd
);
1433 for (i
= 0; i
< nr_frags
; i
++) {
1434 struct skb_frag_struct
*frag
;
1436 frag
= &skb_shinfo(skb
)->frags
[i
];
1438 tpbuf
= GET_TPD_BUFFER(tx_q
, tx_q
->tpd
.produce_idx
);
1439 tpbuf
->length
= frag
->size
;
1440 tpbuf
->dma_addr
= dma_map_page(adpt
->netdev
->dev
.parent
,
1441 frag
->page
.p
, frag
->page_offset
,
1442 tpbuf
->length
, DMA_TO_DEVICE
);
1443 ret
= dma_mapping_error(adpt
->netdev
->dev
.parent
,
1448 TPD_BUFFER_ADDR_L_SET(tpd
, lower_32_bits(tpbuf
->dma_addr
));
1449 TPD_BUFFER_ADDR_H_SET(tpd
, upper_32_bits(tpbuf
->dma_addr
));
1450 TPD_BUF_LEN_SET(tpd
, tpbuf
->length
);
1451 emac_tx_tpd_create(adpt
, tx_q
, tpd
);
1457 emac_tx_tpd_mark_last(adpt
, tx_q
);
1459 /* The last buffer info contain the skb address,
1460 * so it will be freed after unmap
1467 /* One of the memory mappings failed, so undo everything */
1468 tx_q
->tpd
.produce_idx
= first
;
1471 tpbuf
= GET_TPD_BUFFER(tx_q
, first
);
1472 dma_unmap_page(adpt
->netdev
->dev
.parent
, tpbuf
->dma_addr
,
1473 tpbuf
->length
, DMA_TO_DEVICE
);
1474 tpbuf
->dma_addr
= 0;
1477 if (++first
== tx_q
->tpd
.count
)
1484 /* Transmit the packet using specified transmit queue */
1485 int emac_mac_tx_buf_send(struct emac_adapter
*adpt
, struct emac_tx_queue
*tx_q
,
1486 struct sk_buff
*skb
)
1488 struct emac_tpd tpd
;
1491 memset(&tpd
, 0, sizeof(tpd
));
1493 if (emac_tso_csum(adpt
, tx_q
, skb
, &tpd
) != 0) {
1494 dev_kfree_skb_any(skb
);
1495 return NETDEV_TX_OK
;
1498 if (skb_vlan_tag_present(skb
)) {
1501 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb
), tag
);
1502 TPD_CVLAN_TAG_SET(&tpd
, tag
);
1503 TPD_INSTC_SET(&tpd
, 1);
1506 if (skb_network_offset(skb
) != ETH_HLEN
)
1507 TPD_TYP_SET(&tpd
, 1);
1509 emac_tx_fill_tpd(adpt
, tx_q
, skb
, &tpd
);
1511 netdev_sent_queue(adpt
->netdev
, skb
->len
);
1513 /* Make sure the are enough free descriptors to hold one
1514 * maximum-sized SKB. We need one desc for each fragment,
1515 * one for the checksum (emac_tso_csum), one for TSO, and
1516 * and one for the SKB header.
1518 if (emac_tpd_num_free_descs(tx_q
) < (MAX_SKB_FRAGS
+ 3))
1519 netif_stop_queue(adpt
->netdev
);
1521 /* update produce idx */
1522 prod_idx
= (tx_q
->tpd
.produce_idx
<< tx_q
->produce_shift
) &
1524 emac_reg_update32(adpt
->base
+ tx_q
->produce_reg
,
1525 tx_q
->produce_mask
, prod_idx
);
1527 return NETDEV_TX_OK
;