projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'lightnvm/for-next'
[deliverable/linux.git] / drivers / net / ethernet / qualcomm / emac / emac-mac.c
1 /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 */
12
13 /* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support
14 */
15
16 #include <linux/tcp.h>
17 #include <linux/ip.h>
18 #include <linux/ipv6.h>
19 #include <linux/crc32.h>
20 #include <linux/if_vlan.h>
21 #include <linux/jiffies.h>
22 #include <linux/phy.h>
23 #include <linux/of.h>
24 #include <net/ip6_checksum.h>
25 #include "emac.h"
26 #include "emac-sgmii.h"
27
28 /* EMAC base register offsets */
29 #define EMAC_MAC_CTRL 0x001480
30 #define EMAC_WOL_CTRL0 0x0014a0
31 #define EMAC_RSS_KEY0 0x0014b0
32 #define EMAC_H1TPD_BASE_ADDR_LO 0x0014e0
33 #define EMAC_H2TPD_BASE_ADDR_LO 0x0014e4
34 #define EMAC_H3TPD_BASE_ADDR_LO 0x0014e8
35 #define EMAC_INTER_SRAM_PART9 0x001534
36 #define EMAC_DESC_CTRL_0 0x001540
37 #define EMAC_DESC_CTRL_1 0x001544
38 #define EMAC_DESC_CTRL_2 0x001550
39 #define EMAC_DESC_CTRL_10 0x001554
40 #define EMAC_DESC_CTRL_12 0x001558
41 #define EMAC_DESC_CTRL_13 0x00155c
42 #define EMAC_DESC_CTRL_3 0x001560
43 #define EMAC_DESC_CTRL_4 0x001564
44 #define EMAC_DESC_CTRL_5 0x001568
45 #define EMAC_DESC_CTRL_14 0x00156c
46 #define EMAC_DESC_CTRL_15 0x001570
47 #define EMAC_DESC_CTRL_16 0x001574
48 #define EMAC_DESC_CTRL_6 0x001578
49 #define EMAC_DESC_CTRL_8 0x001580
50 #define EMAC_DESC_CTRL_9 0x001584
51 #define EMAC_DESC_CTRL_11 0x001588
52 #define EMAC_TXQ_CTRL_0 0x001590
53 #define EMAC_TXQ_CTRL_1 0x001594
54 #define EMAC_TXQ_CTRL_2 0x001598
55 #define EMAC_RXQ_CTRL_0 0x0015a0
56 #define EMAC_RXQ_CTRL_1 0x0015a4
57 #define EMAC_RXQ_CTRL_2 0x0015a8
58 #define EMAC_RXQ_CTRL_3 0x0015ac
59 #define EMAC_BASE_CPU_NUMBER 0x0015b8
60 #define EMAC_DMA_CTRL 0x0015c0
61 #define EMAC_MAILBOX_0 0x0015e0
62 #define EMAC_MAILBOX_5 0x0015e4
63 #define EMAC_MAILBOX_6 0x0015e8
64 #define EMAC_MAILBOX_13 0x0015ec
65 #define EMAC_MAILBOX_2 0x0015f4
66 #define EMAC_MAILBOX_3 0x0015f8
67 #define EMAC_MAILBOX_11 0x00160c
68 #define EMAC_AXI_MAST_CTRL 0x001610
69 #define EMAC_MAILBOX_12 0x001614
70 #define EMAC_MAILBOX_9 0x001618
71 #define EMAC_MAILBOX_10 0x00161c
72 #define EMAC_ATHR_HEADER_CTRL 0x001620
73 #define EMAC_CLK_GATE_CTRL 0x001814
74 #define EMAC_MISC_CTRL 0x001990
75 #define EMAC_MAILBOX_7 0x0019e0
76 #define EMAC_MAILBOX_8 0x0019e4
77 #define EMAC_MAILBOX_15 0x001bd4
78 #define EMAC_MAILBOX_16 0x001bd8
79
80 /* EMAC_MAC_CTRL */
81 #define SINGLE_PAUSE_MODE 0x10000000
82 #define DEBUG_MODE 0x08000000
83 #define BROAD_EN 0x04000000
84 #define MULTI_ALL 0x02000000
85 #define RX_CHKSUM_EN 0x01000000
86 #define HUGE 0x00800000
87 #define SPEED(x) (((x) & 0x3) << 20)
88 #define SPEED_MASK SPEED(0x3)
89 #define SIMR 0x00080000
90 #define TPAUSE 0x00010000
91 #define PROM_MODE 0x00008000
92 #define VLAN_STRIP 0x00004000
93 #define PRLEN_BMSK 0x00003c00
94 #define PRLEN_SHFT 10
95 #define HUGEN 0x00000200
96 #define FLCHK 0x00000100
97 #define PCRCE 0x00000080
98 #define CRCE 0x00000040
99 #define FULLD 0x00000020
100 #define MAC_LP_EN 0x00000010
101 #define RXFC 0x00000008
102 #define TXFC 0x00000004
103 #define RXEN 0x00000002
104 #define TXEN 0x00000001
105
106
107 /* EMAC_WOL_CTRL0 */
108 #define LK_CHG_PME 0x20
109 #define LK_CHG_EN 0x10
110 #define MG_FRAME_PME 0x8
111 #define MG_FRAME_EN 0x4
112 #define WK_FRAME_EN 0x1
113
114 /* EMAC_DESC_CTRL_3 */
115 #define RFD_RING_SIZE_BMSK 0xfff
116
117 /* EMAC_DESC_CTRL_4 */
118 #define RX_BUFFER_SIZE_BMSK 0xffff
119
120 /* EMAC_DESC_CTRL_6 */
121 #define RRD_RING_SIZE_BMSK 0xfff
122
123 /* EMAC_DESC_CTRL_9 */
124 #define TPD_RING_SIZE_BMSK 0xffff
125
126 /* EMAC_TXQ_CTRL_0 */
127 #define NUM_TXF_BURST_PREF_BMSK 0xffff0000
128 #define NUM_TXF_BURST_PREF_SHFT 16
129 #define LS_8023_SP 0x80
130 #define TXQ_MODE 0x40
131 #define TXQ_EN 0x20
132 #define IP_OP_SP 0x10
133 #define NUM_TPD_BURST_PREF_BMSK 0xf
134 #define NUM_TPD_BURST_PREF_SHFT 0
135
136 /* EMAC_TXQ_CTRL_1 */
137 #define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK 0x7ff
138
139 /* EMAC_TXQ_CTRL_2 */
140 #define TXF_HWM_BMSK 0xfff0000
141 #define TXF_LWM_BMSK 0xfff
142
143 /* EMAC_RXQ_CTRL_0 */
144 #define RXQ_EN BIT(31)
145 #define CUT_THRU_EN BIT(30)
146 #define RSS_HASH_EN BIT(29)
147 #define NUM_RFD_BURST_PREF_BMSK 0x3f00000
148 #define NUM_RFD_BURST_PREF_SHFT 20
149 #define IDT_TABLE_SIZE_BMSK 0x1ff00
150 #define IDT_TABLE_SIZE_SHFT 8
151 #define SP_IPV6 0x80
152
153 /* EMAC_RXQ_CTRL_1 */
154 #define JUMBO_1KAH_BMSK 0xf000
155 #define JUMBO_1KAH_SHFT 12
156 #define RFD_PREF_LOW_TH 0x10
157 #define RFD_PREF_LOW_THRESHOLD_BMSK 0xfc0
158 #define RFD_PREF_LOW_THRESHOLD_SHFT 6
159 #define RFD_PREF_UP_TH 0x10
160 #define RFD_PREF_UP_THRESHOLD_BMSK 0x3f
161 #define RFD_PREF_UP_THRESHOLD_SHFT 0
162
163 /* EMAC_RXQ_CTRL_2 */
164 #define RXF_DOF_THRESFHOLD 0x1a0
165 #define RXF_DOF_THRESHOLD_BMSK 0xfff0000
166 #define RXF_DOF_THRESHOLD_SHFT 16
167 #define RXF_UOF_THRESFHOLD 0xbe
168 #define RXF_UOF_THRESHOLD_BMSK 0xfff
169 #define RXF_UOF_THRESHOLD_SHFT 0
170
171 /* EMAC_RXQ_CTRL_3 */
172 #define RXD_TIMER_BMSK 0xffff0000
173 #define RXD_THRESHOLD_BMSK 0xfff
174 #define RXD_THRESHOLD_SHFT 0
175
176 /* EMAC_DMA_CTRL */
177 #define DMAW_DLY_CNT_BMSK 0xf0000
178 #define DMAW_DLY_CNT_SHFT 16
179 #define DMAR_DLY_CNT_BMSK 0xf800
180 #define DMAR_DLY_CNT_SHFT 11
181 #define DMAR_REQ_PRI 0x400
182 #define REGWRBLEN_BMSK 0x380
183 #define REGWRBLEN_SHFT 7
184 #define REGRDBLEN_BMSK 0x70
185 #define REGRDBLEN_SHFT 4
186 #define OUT_ORDER_MODE 0x4
187 #define ENH_ORDER_MODE 0x2
188 #define IN_ORDER_MODE 0x1
189
190 /* EMAC_MAILBOX_13 */
191 #define RFD3_PROC_IDX_BMSK 0xfff0000
192 #define RFD3_PROC_IDX_SHFT 16
193 #define RFD3_PROD_IDX_BMSK 0xfff
194 #define RFD3_PROD_IDX_SHFT 0
195
196 /* EMAC_MAILBOX_2 */
197 #define NTPD_CONS_IDX_BMSK 0xffff0000
198 #define NTPD_CONS_IDX_SHFT 16
199
200 /* EMAC_MAILBOX_3 */
201 #define RFD0_CONS_IDX_BMSK 0xfff
202 #define RFD0_CONS_IDX_SHFT 0
203
204 /* EMAC_MAILBOX_11 */
205 #define H3TPD_PROD_IDX_BMSK 0xffff0000
206 #define H3TPD_PROD_IDX_SHFT 16
207
208 /* EMAC_AXI_MAST_CTRL */
209 #define DATA_BYTE_SWAP 0x8
210 #define MAX_BOUND 0x2
211 #define MAX_BTYPE 0x1
212
213 /* EMAC_MAILBOX_12 */
214 #define H3TPD_CONS_IDX_BMSK 0xffff0000
215 #define H3TPD_CONS_IDX_SHFT 16
216
217 /* EMAC_MAILBOX_9 */
218 #define H2TPD_PROD_IDX_BMSK 0xffff
219 #define H2TPD_PROD_IDX_SHFT 0
220
221 /* EMAC_MAILBOX_10 */
222 #define H1TPD_CONS_IDX_BMSK 0xffff0000
223 #define H1TPD_CONS_IDX_SHFT 16
224 #define H2TPD_CONS_IDX_BMSK 0xffff
225 #define H2TPD_CONS_IDX_SHFT 0
226
227 /* EMAC_ATHR_HEADER_CTRL */
228 #define HEADER_CNT_EN 0x2
229 #define HEADER_ENABLE 0x1
230
231 /* EMAC_MAILBOX_0 */
232 #define RFD0_PROC_IDX_BMSK 0xfff0000
233 #define RFD0_PROC_IDX_SHFT 16
234 #define RFD0_PROD_IDX_BMSK 0xfff
235 #define RFD0_PROD_IDX_SHFT 0
236
237 /* EMAC_MAILBOX_5 */
238 #define RFD1_PROC_IDX_BMSK 0xfff0000
239 #define RFD1_PROC_IDX_SHFT 16
240 #define RFD1_PROD_IDX_BMSK 0xfff
241 #define RFD1_PROD_IDX_SHFT 0
242
243 /* EMAC_MISC_CTRL */
244 #define RX_UNCPL_INT_EN 0x1
245
246 /* EMAC_MAILBOX_7 */
247 #define RFD2_CONS_IDX_BMSK 0xfff0000
248 #define RFD2_CONS_IDX_SHFT 16
249 #define RFD1_CONS_IDX_BMSK 0xfff
250 #define RFD1_CONS_IDX_SHFT 0
251
252 /* EMAC_MAILBOX_8 */
253 #define RFD3_CONS_IDX_BMSK 0xfff
254 #define RFD3_CONS_IDX_SHFT 0
255
256 /* EMAC_MAILBOX_15 */
257 #define NTPD_PROD_IDX_BMSK 0xffff
258 #define NTPD_PROD_IDX_SHFT 0
259
260 /* EMAC_MAILBOX_16 */
261 #define H1TPD_PROD_IDX_BMSK 0xffff
262 #define H1TPD_PROD_IDX_SHFT 0
263
264 #define RXQ0_RSS_HSTYP_IPV6_TCP_EN 0x20
265 #define RXQ0_RSS_HSTYP_IPV6_EN 0x10
266 #define RXQ0_RSS_HSTYP_IPV4_TCP_EN 0x8
267 #define RXQ0_RSS_HSTYP_IPV4_EN 0x4
268
269 /* EMAC_EMAC_WRAPPER_TX_TS_INX */
270 #define EMAC_WRAPPER_TX_TS_EMPTY BIT(31)
271 #define EMAC_WRAPPER_TX_TS_INX_BMSK 0xffff
272
273 struct emac_skb_cb {
274 u32 tpd_idx;
275 unsigned long jiffies;
276 };
277
278 #define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb)
279 #define EMAC_RSS_IDT_SIZE 256
280 #define JUMBO_1KAH 0x4
281 #define RXD_TH 0x100
282 #define EMAC_TPD_LAST_FRAGMENT 0x80000000
283 #define EMAC_TPD_TSTAMP_SAVE 0x80000000
284
285 /* EMAC Errors in emac_rrd.word[3] */
286 #define EMAC_RRD_L4F BIT(14)
287 #define EMAC_RRD_IPF BIT(15)
288 #define EMAC_RRD_CRC BIT(21)
289 #define EMAC_RRD_FAE BIT(22)
290 #define EMAC_RRD_TRN BIT(23)
291 #define EMAC_RRD_RNT BIT(24)
292 #define EMAC_RRD_INC BIT(25)
293 #define EMAC_RRD_FOV BIT(29)
294 #define EMAC_RRD_LEN BIT(30)
295
296 /* Error bits that will result in a received frame being discarded */
297 #define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \
298 EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \
299 EMAC_RRD_FOV | EMAC_RRD_LEN)
300 #define EMAC_RRD_STATS_DW_IDX 3
301
302 #define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX)))
303 #define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX)))
304 #define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX)))
305
306 #define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)]))
307 #define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)]))
308
309 #define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD 8
310
311 #define ISR_RX_PKT (\
312 RX_PKT_INT0 |\
313 RX_PKT_INT1 |\
314 RX_PKT_INT2 |\
315 RX_PKT_INT3)
316
317 #define EMAC_MAC_IRQ_RES "core0"
318
319 void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr)
320 {
321 u32 crc32, bit, reg, mta;
322
323 /* Calculate the CRC of the MAC address */
324 crc32 = ether_crc(ETH_ALEN, addr);
325
326 /* The HASH Table is an array of 2 32-bit registers. It is
327 * treated like an array of 64 bits (BitArray[hash_value]).
328 * Use the upper 6 bits of the above CRC as the hash value.
329 */
330 reg = (crc32 >> 31) & 0x1;
331 bit = (crc32 >> 26) & 0x1F;
332
333 mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
334 mta |= BIT(bit);
335 writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
336 }
337
338 void emac_mac_multicast_addr_clear(struct emac_adapter *adpt)
339 {
340 writel(0, adpt->base + EMAC_HASH_TAB_REG0);
341 writel(0, adpt->base + EMAC_HASH_TAB_REG1);
342 }
343
344 /* definitions for RSS */
345 #define EMAC_RSS_KEY(_i, _type) \
346 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))
347 #define EMAC_RSS_TBL(_i, _type) \
348 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))
349
350 /* Config MAC modes */
351 void emac_mac_mode_config(struct emac_adapter *adpt)
352 {
353 struct net_device *netdev = adpt->netdev;
354 u32 mac;
355
356 mac = readl(adpt->base + EMAC_MAC_CTRL);
357 mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN);
358
359 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
360 mac |= VLAN_STRIP;
361
362 if (netdev->flags & IFF_PROMISC)
363 mac |= PROM_MODE;
364
365 if (netdev->flags & IFF_ALLMULTI)
366 mac |= MULTI_ALL;
367
368 writel(mac, adpt->base + EMAC_MAC_CTRL);
369 }
370
371 /* Config descriptor rings */
372 static void emac_mac_dma_rings_config(struct emac_adapter *adpt)
373 {
374 static const unsigned short tpd_q_offset[] = {
375 EMAC_DESC_CTRL_8, EMAC_H1TPD_BASE_ADDR_LO,
376 EMAC_H2TPD_BASE_ADDR_LO, EMAC_H3TPD_BASE_ADDR_LO};
377 static const unsigned short rfd_q_offset[] = {
378 EMAC_DESC_CTRL_2, EMAC_DESC_CTRL_10,
379 EMAC_DESC_CTRL_12, EMAC_DESC_CTRL_13};
380 static const unsigned short rrd_q_offset[] = {
381 EMAC_DESC_CTRL_5, EMAC_DESC_CTRL_14,
382 EMAC_DESC_CTRL_15, EMAC_DESC_CTRL_16};
383
384 /* TPD (Transmit Packet Descriptor) */
385 writel(upper_32_bits(adpt->tx_q.tpd.dma_addr),
386 adpt->base + EMAC_DESC_CTRL_1);
387
388 writel(lower_32_bits(adpt->tx_q.tpd.dma_addr),
389 adpt->base + tpd_q_offset[0]);
390
391 writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK,
392 adpt->base + EMAC_DESC_CTRL_9);
393
394 /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */
395 writel(upper_32_bits(adpt->rx_q.rfd.dma_addr),
396 adpt->base + EMAC_DESC_CTRL_0);
397
398 writel(lower_32_bits(adpt->rx_q.rfd.dma_addr),
399 adpt->base + rfd_q_offset[0]);
400 writel(lower_32_bits(adpt->rx_q.rrd.dma_addr),
401 adpt->base + rrd_q_offset[0]);
402
403 writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK,
404 adpt->base + EMAC_DESC_CTRL_3);
405 writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK,
406 adpt->base + EMAC_DESC_CTRL_6);
407
408 writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK,
409 adpt->base + EMAC_DESC_CTRL_4);
410
411 writel(0, adpt->base + EMAC_DESC_CTRL_11);
412
413 /* Load all of the base addresses above and ensure that triggering HW to
414 * read ring pointers is flushed
415 */
416 writel(1, adpt->base + EMAC_INTER_SRAM_PART9);
417 }
418
419 /* Config transmit parameters */
420 static void emac_mac_tx_config(struct emac_adapter *adpt)
421 {
422 u32 val;
423
424 writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) &
425 JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1);
426
427 val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) &
428 NUM_TPD_BURST_PREF_BMSK;
429
430 val |= TXQ_MODE | LS_8023_SP;
431 val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) &
432 NUM_TXF_BURST_PREF_BMSK;
433
434 writel(val, adpt->base + EMAC_TXQ_CTRL_0);
435 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2,
436 (TXF_HWM_BMSK | TXF_LWM_BMSK), 0);
437 }
438
439 /* Config receive parameters */
440 static void emac_mac_rx_config(struct emac_adapter *adpt)
441 {
442 u32 val;
443
444 val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) &
445 NUM_RFD_BURST_PREF_BMSK;
446 val |= (SP_IPV6 | CUT_THRU_EN);
447
448 writel(val, adpt->base + EMAC_RXQ_CTRL_0);
449
450 val = readl(adpt->base + EMAC_RXQ_CTRL_1);
451 val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK |
452 RFD_PREF_UP_THRESHOLD_BMSK);
453 val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) |
454 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) |
455 (RFD_PREF_UP_TH << RFD_PREF_UP_THRESHOLD_SHFT);
456 writel(val, adpt->base + EMAC_RXQ_CTRL_1);
457
458 val = readl(adpt->base + EMAC_RXQ_CTRL_2);
459 val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK);
460 val |= (RXF_DOF_THRESFHOLD << RXF_DOF_THRESHOLD_SHFT) |
461 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT);
462 writel(val, adpt->base + EMAC_RXQ_CTRL_2);
463
464 val = readl(adpt->base + EMAC_RXQ_CTRL_3);
465 val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK);
466 val |= RXD_TH << RXD_THRESHOLD_SHFT;
467 writel(val, adpt->base + EMAC_RXQ_CTRL_3);
468 }
469
470 /* Config dma */
471 static void emac_mac_dma_config(struct emac_adapter *adpt)
472 {
473 u32 dma_ctrl = DMAR_REQ_PRI;
474
475 switch (adpt->dma_order) {
476 case emac_dma_ord_in:
477 dma_ctrl |= IN_ORDER_MODE;
478 break;
479 case emac_dma_ord_enh:
480 dma_ctrl |= ENH_ORDER_MODE;
481 break;
482 case emac_dma_ord_out:
483 dma_ctrl |= OUT_ORDER_MODE;
484 break;
485 default:
486 break;
487 }
488
489 dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) &
490 REGRDBLEN_BMSK;
491 dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) &
492 REGWRBLEN_BMSK;
493 dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) &
494 DMAR_DLY_CNT_BMSK;
495 dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) &
496 DMAW_DLY_CNT_BMSK;
497
498 /* config DMA and ensure that configuration is flushed to HW */
499 writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL);
500 }
501
502 /* set MAC address */
503 static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr)
504 {
505 u32 sta;
506
507 /* for example: 00-A0-C6-11-22-33
508 * 0<-->C6112233, 1<-->00A0.
509 */
510
511 /* low 32bit word */
512 sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) |
513 (((u32)addr[4]) << 8) | (((u32)addr[5]));
514 writel(sta, adpt->base + EMAC_MAC_STA_ADDR0);
515
516 /* hight 32bit word */
517 sta = (((u32)addr[0]) << 8) | (u32)addr[1];
518 writel(sta, adpt->base + EMAC_MAC_STA_ADDR1);
519 }
520
521 static void emac_mac_config(struct emac_adapter *adpt)
522 {
523 struct net_device *netdev = adpt->netdev;
524 unsigned int max_frame;
525 u32 val;
526
527 emac_set_mac_address(adpt, netdev->dev_addr);
528
529 max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
530 adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?
531 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;
532
533 emac_mac_dma_rings_config(adpt);
534
535 writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN,
536 adpt->base + EMAC_MAX_FRAM_LEN_CTRL);
537
538 emac_mac_tx_config(adpt);
539 emac_mac_rx_config(adpt);
540 emac_mac_dma_config(adpt);
541
542 val = readl(adpt->base + EMAC_AXI_MAST_CTRL);
543 val &= ~(DATA_BYTE_SWAP | MAX_BOUND);
544 val |= MAX_BTYPE;
545 writel(val, adpt->base + EMAC_AXI_MAST_CTRL);
546 writel(0, adpt->base + EMAC_CLK_GATE_CTRL);
547 writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL);
548 }
549
550 void emac_mac_reset(struct emac_adapter *adpt)
551 {
552 emac_mac_stop(adpt);
553
554 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST);
555 usleep_range(100, 150); /* reset may take up to 100usec */
556
557 /* interrupt clear-on-read */
558 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN);
559 }
560
561 void emac_mac_start(struct emac_adapter *adpt)
562 {
563 struct phy_device *phydev = adpt->phydev;
564 u32 mac, csr1;
565
566 /* enable tx queue */
567 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN);
568
569 /* enable rx queue */
570 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN);
571
572 /* enable mac control */
573 mac = readl(adpt->base + EMAC_MAC_CTRL);
574 csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
575
576 mac |= TXEN | RXEN; /* enable RX/TX */
577
578 /* We don't have ethtool support yet, so force flow-control mode
579 * to 'full' always.
580 */
581 mac |= TXFC | RXFC;
582
583 /* setup link speed */
584 mac &= ~SPEED_MASK;
585 if (phydev->speed == SPEED_1000) {
586 mac |= SPEED(2);
587 csr1 |= FREQ_MODE;
588 } else {
589 mac |= SPEED(1);
590 csr1 &= ~FREQ_MODE;
591 }
592
593 if (phydev->duplex == DUPLEX_FULL)
594 mac |= FULLD;
595 else
596 mac &= ~FULLD;
597
598 /* other parameters */
599 mac |= (CRCE | PCRCE);
600 mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK);
601 mac |= BROAD_EN;
602 mac |= FLCHK;
603 mac &= ~RX_CHKSUM_EN;
604 mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL |
605 DEBUG_MODE | SINGLE_PAUSE_MODE);
606
607 writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
608
609 writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL);
610
611 /* enable interrupt read clear, low power sleep mode and
612 * the irq moderators
613 */
614
615 writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT);
616 writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN |
617 IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL);
618
619 emac_mac_mode_config(adpt);
620
621 emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL,
622 (HEADER_ENABLE | HEADER_CNT_EN), 0);
623
624 emac_reg_update32(adpt->csr + EMAC_EMAC_WRAPPER_CSR2, 0, WOL_EN);
625 }
626
627 void emac_mac_stop(struct emac_adapter *adpt)
628 {
629 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0);
630 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0);
631 emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0);
632 usleep_range(1000, 1050); /* stopping mac may take upto 1msec */
633 }
634
635 /* Free all descriptors of given transmit queue */
636 static void emac_tx_q_descs_free(struct emac_adapter *adpt)
637 {
638 struct emac_tx_queue *tx_q = &adpt->tx_q;
639 unsigned int i;
640 size_t size;
641
642 /* ring already cleared, nothing to do */
643 if (!tx_q->tpd.tpbuff)
644 return;
645
646 for (i = 0; i < tx_q->tpd.count; i++) {
647 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i);
648
649 if (tpbuf->dma_addr) {
650 dma_unmap_single(adpt->netdev->dev.parent,
651 tpbuf->dma_addr, tpbuf->length,
652 DMA_TO_DEVICE);
653 tpbuf->dma_addr = 0;
654 }
655 if (tpbuf->skb) {
656 dev_kfree_skb_any(tpbuf->skb);
657 tpbuf->skb = NULL;
658 }
659 }
660
661 size = sizeof(struct emac_buffer) * tx_q->tpd.count;
662 memset(tx_q->tpd.tpbuff, 0, size);
663
664 /* clear the descriptor ring */
665 memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size);
666
667 tx_q->tpd.consume_idx = 0;
668 tx_q->tpd.produce_idx = 0;
669 }
670
671 /* Free all descriptors of given receive queue */
672 static void emac_rx_q_free_descs(struct emac_adapter *adpt)
673 {
674 struct device *dev = adpt->netdev->dev.parent;
675 struct emac_rx_queue *rx_q = &adpt->rx_q;
676 unsigned int i;
677 size_t size;
678
679 /* ring already cleared, nothing to do */
680 if (!rx_q->rfd.rfbuff)
681 return;
682
683 for (i = 0; i < rx_q->rfd.count; i++) {
684 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i);
685
686 if (rfbuf->dma_addr) {
687 dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length,
688 DMA_FROM_DEVICE);
689 rfbuf->dma_addr = 0;
690 }
691 if (rfbuf->skb) {
692 dev_kfree_skb(rfbuf->skb);
693 rfbuf->skb = NULL;
694 }
695 }
696
697 size = sizeof(struct emac_buffer) * rx_q->rfd.count;
698 memset(rx_q->rfd.rfbuff, 0, size);
699
700 /* clear the descriptor rings */
701 memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size);
702 rx_q->rrd.produce_idx = 0;
703 rx_q->rrd.consume_idx = 0;
704
705 memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size);
706 rx_q->rfd.produce_idx = 0;
707 rx_q->rfd.consume_idx = 0;
708 }
709
710 /* Free all buffers associated with given transmit queue */
711 static void emac_tx_q_bufs_free(struct emac_adapter *adpt)
712 {
713 struct emac_tx_queue *tx_q = &adpt->tx_q;
714
715 emac_tx_q_descs_free(adpt);
716
717 kfree(tx_q->tpd.tpbuff);
718 tx_q->tpd.tpbuff = NULL;
719 tx_q->tpd.v_addr = NULL;
720 tx_q->tpd.dma_addr = 0;
721 tx_q->tpd.size = 0;
722 }
723
724 /* Allocate TX descriptor ring for the given transmit queue */
725 static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,
726 struct emac_tx_queue *tx_q)
727 {
728 struct emac_ring_header *ring_header = &adpt->ring_header;
729 size_t size;
730
731 size = sizeof(struct emac_buffer) * tx_q->tpd.count;
732 tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL);
733 if (!tx_q->tpd.tpbuff)
734 return -ENOMEM;
735
736 tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4);
737 tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used;
738 tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used;
739 ring_header->used += ALIGN(tx_q->tpd.size, 8);
740 tx_q->tpd.produce_idx = 0;
741 tx_q->tpd.consume_idx = 0;
742
743 return 0;
744 }
745
746 /* Free all buffers associated with given transmit queue */
747 static void emac_rx_q_bufs_free(struct emac_adapter *adpt)
748 {
749 struct emac_rx_queue *rx_q = &adpt->rx_q;
750
751 emac_rx_q_free_descs(adpt);
752
753 kfree(rx_q->rfd.rfbuff);
754 rx_q->rfd.rfbuff = NULL;
755
756 rx_q->rfd.v_addr = NULL;
757 rx_q->rfd.dma_addr = 0;
758 rx_q->rfd.size = 0;
759
760 rx_q->rrd.v_addr = NULL;
761 rx_q->rrd.dma_addr = 0;
762 rx_q->rrd.size = 0;
763 }
764
765 /* Allocate RX descriptor rings for the given receive queue */
766 static int emac_rx_descs_alloc(struct emac_adapter *adpt)
767 {
768 struct emac_ring_header *ring_header = &adpt->ring_header;
769 struct emac_rx_queue *rx_q = &adpt->rx_q;
770 size_t size;
771
772 size = sizeof(struct emac_buffer) * rx_q->rfd.count;
773 rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL);
774 if (!rx_q->rfd.rfbuff)
775 return -ENOMEM;
776
777 rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4);
778 rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4);
779
780 rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used;
781 rx_q->rrd.v_addr = ring_header->v_addr + ring_header->used;
782 ring_header->used += ALIGN(rx_q->rrd.size, 8);
783
784 rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used;
785 rx_q->rfd.v_addr = ring_header->v_addr + ring_header->used;
786 ring_header->used += ALIGN(rx_q->rfd.size, 8);
787
788 rx_q->rrd.produce_idx = 0;
789 rx_q->rrd.consume_idx = 0;
790
791 rx_q->rfd.produce_idx = 0;
792 rx_q->rfd.consume_idx = 0;
793
794 return 0;
795 }
796
797 /* Allocate all TX and RX descriptor rings */
798 int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt)
799 {
800 struct emac_ring_header *ring_header = &adpt->ring_header;
801 struct device *dev = adpt->netdev->dev.parent;
802 unsigned int num_tx_descs = adpt->tx_desc_cnt;
803 unsigned int num_rx_descs = adpt->rx_desc_cnt;
804 int ret;
805
806 adpt->tx_q.tpd.count = adpt->tx_desc_cnt;
807
808 adpt->rx_q.rrd.count = adpt->rx_desc_cnt;
809 adpt->rx_q.rfd.count = adpt->rx_desc_cnt;
810
811 /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,
812 * hence the additional padding bytes are allocated.
813 */
814 ring_header->size = num_tx_descs * (adpt->tpd_size * 4) +
815 num_rx_descs * (adpt->rfd_size * 4) +
816 num_rx_descs * (adpt->rrd_size * 4) +
817 8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */
818
819 ring_header->used = 0;
820 ring_header->v_addr = dma_zalloc_coherent(dev, ring_header->size,
821 &ring_header->dma_addr,
822 GFP_KERNEL);
823 if (!ring_header->v_addr)
824 return -ENOMEM;
825
826 ring_header->used = ALIGN(ring_header->dma_addr, 8) -
827 ring_header->dma_addr;
828
829 ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q);
830 if (ret) {
831 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n");
832 goto err_alloc_tx;
833 }
834
835 ret = emac_rx_descs_alloc(adpt);
836 if (ret) {
837 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n");
838 goto err_alloc_rx;
839 }
840
841 return 0;
842
843 err_alloc_rx:
844 emac_tx_q_bufs_free(adpt);
845 err_alloc_tx:
846 dma_free_coherent(dev, ring_header->size,
847 ring_header->v_addr, ring_header->dma_addr);
848
849 ring_header->v_addr = NULL;
850 ring_header->dma_addr = 0;
851 ring_header->size = 0;
852 ring_header->used = 0;
853
854 return ret;
855 }
856
857 /* Free all TX and RX descriptor rings */
858 void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt)
859 {
860 struct emac_ring_header *ring_header = &adpt->ring_header;
861 struct device *dev = adpt->netdev->dev.parent;
862
863 emac_tx_q_bufs_free(adpt);
864 emac_rx_q_bufs_free(adpt);
865
866 dma_free_coherent(dev, ring_header->size,
867 ring_header->v_addr, ring_header->dma_addr);
868
869 ring_header->v_addr = NULL;
870 ring_header->dma_addr = 0;
871 ring_header->size = 0;
872 ring_header->used = 0;
873 }
874
875 /* Initialize descriptor rings */
876 static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt)
877 {
878 unsigned int i;
879
880 adpt->tx_q.tpd.produce_idx = 0;
881 adpt->tx_q.tpd.consume_idx = 0;
882 for (i = 0; i < adpt->tx_q.tpd.count; i++)
883 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0;
884
885 adpt->rx_q.rrd.produce_idx = 0;
886 adpt->rx_q.rrd.consume_idx = 0;
887 adpt->rx_q.rfd.produce_idx = 0;
888 adpt->rx_q.rfd.consume_idx = 0;
889 for (i = 0; i < adpt->rx_q.rfd.count; i++)
890 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0;
891 }
892
893 /* Produce new receive free descriptor */
894 static void emac_mac_rx_rfd_create(struct emac_adapter *adpt,
895 struct emac_rx_queue *rx_q,
896 dma_addr_t addr)
897 {
898 u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx);
899
900 *(hw_rfd++) = lower_32_bits(addr);
901 *hw_rfd = upper_32_bits(addr);
902
903 if (++rx_q->rfd.produce_idx == rx_q->rfd.count)
904 rx_q->rfd.produce_idx = 0;
905 }
906
907 /* Fill up receive queue's RFD with preallocated receive buffers */
908 static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,
909 struct emac_rx_queue *rx_q)
910 {
911 struct emac_buffer *curr_rxbuf;
912 struct emac_buffer *next_rxbuf;
913 unsigned int count = 0;
914 u32 next_produce_idx;
915
916 next_produce_idx = rx_q->rfd.produce_idx + 1;
917 if (next_produce_idx == rx_q->rfd.count)
918 next_produce_idx = 0;
919
920 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
921 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
922
923 /* this always has a blank rx_buffer*/
924 while (!next_rxbuf->dma_addr) {
925 struct sk_buff *skb;
926 int ret;
927
928 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size);
929 if (!skb)
930 break;
931
932 curr_rxbuf->dma_addr =
933 dma_map_single(adpt->netdev->dev.parent, skb->data,
934 curr_rxbuf->length, DMA_FROM_DEVICE);
935 ret = dma_mapping_error(adpt->netdev->dev.parent,
936 curr_rxbuf->dma_addr);
937 if (ret) {
938 dev_kfree_skb(skb);
939 break;
940 }
941 curr_rxbuf->skb = skb;
942 curr_rxbuf->length = adpt->rxbuf_size;
943
944 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr);
945 next_produce_idx = rx_q->rfd.produce_idx + 1;
946 if (next_produce_idx == rx_q->rfd.count)
947 next_produce_idx = 0;
948
949 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
950 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
951 count++;
952 }
953
954 if (count) {
955 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) &
956 rx_q->produce_mask;
957 emac_reg_update32(adpt->base + rx_q->produce_reg,
958 rx_q->produce_mask, prod_idx);
959 }
960 }
961
962 static void emac_adjust_link(struct net_device *netdev)
963 {
964 struct emac_adapter *adpt = netdev_priv(netdev);
965 struct phy_device *phydev = netdev->phydev;
966
967 if (phydev->link)
968 emac_mac_start(adpt);
969 else
970 emac_mac_stop(adpt);
971
972 phy_print_status(phydev);
973 }
974
975 /* Bringup the interface/HW */
976 int emac_mac_up(struct emac_adapter *adpt)
977 {
978 struct net_device *netdev = adpt->netdev;
979 struct emac_irq *irq = &adpt->irq;
980 int ret;
981
982 emac_mac_rx_tx_ring_reset_all(adpt);
983 emac_mac_config(adpt);
984
985 ret = request_irq(irq->irq, emac_isr, 0, EMAC_MAC_IRQ_RES, irq);
986 if (ret) {
987 netdev_err(adpt->netdev, "could not request %s irq\n",
988 EMAC_MAC_IRQ_RES);
989 return ret;
990 }
991
992 emac_mac_rx_descs_refill(adpt, &adpt->rx_q);
993
994 ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link,
995 PHY_INTERFACE_MODE_SGMII);
996 if (ret) {
997 netdev_err(adpt->netdev, "could not connect phy\n");
998 free_irq(irq->irq, irq);
999 return ret;
1000 }
1001
1002 /* enable mac irq */
1003 writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS);
1004 writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK);
1005
1006 adpt->phydev->irq = PHY_IGNORE_INTERRUPT;
1007 phy_start(adpt->phydev);
1008
1009 napi_enable(&adpt->rx_q.napi);
1010 netif_start_queue(netdev);
1011
1012 return 0;
1013 }
1014
1015 /* Bring down the interface/HW */
1016 void emac_mac_down(struct emac_adapter *adpt)
1017 {
1018 struct net_device *netdev = adpt->netdev;
1019
1020 netif_stop_queue(netdev);
1021 napi_disable(&adpt->rx_q.napi);
1022
1023 phy_stop(adpt->phydev);
1024 phy_disconnect(adpt->phydev);
1025
1026 /* disable mac irq */
1027 writel(DIS_INT, adpt->base + EMAC_INT_STATUS);
1028 writel(0, adpt->base + EMAC_INT_MASK);
1029 synchronize_irq(adpt->irq.irq);
1030 free_irq(adpt->irq.irq, &adpt->irq);
1031
1032 emac_mac_reset(adpt);
1033
1034 emac_tx_q_descs_free(adpt);
1035 netdev_reset_queue(adpt->netdev);
1036 emac_rx_q_free_descs(adpt);
1037 }
1038
1039 /* Consume next received packet descriptor */
1040 static bool emac_rx_process_rrd(struct emac_adapter *adpt,
1041 struct emac_rx_queue *rx_q,
1042 struct emac_rrd *rrd)
1043 {
1044 u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx);
1045
1046 rrd->word[3] = *(hw_rrd + 3);
1047
1048 if (!RRD_UPDT(rrd))
1049 return false;
1050
1051 rrd->word[4] = 0;
1052 rrd->word[5] = 0;
1053
1054 rrd->word[0] = *(hw_rrd++);
1055 rrd->word[1] = *(hw_rrd++);
1056 rrd->word[2] = *(hw_rrd++);
1057
1058 if (unlikely(RRD_NOR(rrd) != 1)) {
1059 netdev_err(adpt->netdev,
1060 "error: multi-RFD not support yet! nor:%lu\n",
1061 RRD_NOR(rrd));
1062 }
1063
1064 /* mark rrd as processed */
1065 RRD_UPDT_SET(rrd, 0);
1066 *hw_rrd = rrd->word[3];
1067
1068 if (++rx_q->rrd.consume_idx == rx_q->rrd.count)
1069 rx_q->rrd.consume_idx = 0;
1070
1071 return true;
1072 }
1073
1074 /* Produce new transmit descriptor */
1075 static void emac_tx_tpd_create(struct emac_adapter *adpt,
1076 struct emac_tx_queue *tx_q, struct emac_tpd *tpd)
1077 {
1078 u32 *hw_tpd;
1079
1080 tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx;
1081 hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx);
1082
1083 if (++tx_q->tpd.produce_idx == tx_q->tpd.count)
1084 tx_q->tpd.produce_idx = 0;
1085
1086 *(hw_tpd++) = tpd->word[0];
1087 *(hw_tpd++) = tpd->word[1];
1088 *(hw_tpd++) = tpd->word[2];
1089 *hw_tpd = tpd->word[3];
1090 }
1091
1092 /* Mark the last transmit descriptor as such (for the transmit packet) */
1093 static void emac_tx_tpd_mark_last(struct emac_adapter *adpt,
1094 struct emac_tx_queue *tx_q)
1095 {
1096 u32 *hw_tpd =
1097 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx);
1098 u32 tmp_tpd;
1099
1100 tmp_tpd = *(hw_tpd + 1);
1101 tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;
1102 *(hw_tpd + 1) = tmp_tpd;
1103 }
1104
1105 static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd)
1106 {
1107 struct emac_buffer *rfbuf = rx_q->rfd.rfbuff;
1108 u32 consume_idx = RRD_SI(rrd);
1109 unsigned int i;
1110
1111 for (i = 0; i < RRD_NOR(rrd); i++) {
1112 rfbuf[consume_idx].skb = NULL;
1113 if (++consume_idx == rx_q->rfd.count)
1114 consume_idx = 0;
1115 }
1116
1117 rx_q->rfd.consume_idx = consume_idx;
1118 rx_q->rfd.process_idx = consume_idx;
1119 }
1120
1121 /* Push the received skb to upper layers */
1122 static void emac_receive_skb(struct emac_rx_queue *rx_q,
1123 struct sk_buff *skb,
1124 u16 vlan_tag, bool vlan_flag)
1125 {
1126 if (vlan_flag) {
1127 u16 vlan;
1128
1129 EMAC_TAG_TO_VLAN(vlan_tag, vlan);
1130 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
1131 }
1132
1133 napi_gro_receive(&rx_q->napi, skb);
1134 }
1135
1136 /* Process receive event */
1137 void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,
1138 int *num_pkts, int max_pkts)
1139 {
1140 u32 proc_idx, hw_consume_idx, num_consume_pkts;
1141 struct net_device *netdev = adpt->netdev;
1142 struct emac_buffer *rfbuf;
1143 unsigned int count = 0;
1144 struct emac_rrd rrd;
1145 struct sk_buff *skb;
1146 u32 reg;
1147
1148 reg = readl_relaxed(adpt->base + rx_q->consume_reg);
1149
1150 hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift;
1151 num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ?
1152 (hw_consume_idx - rx_q->rrd.consume_idx) :
1153 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx);
1154
1155 do {
1156 if (!num_consume_pkts)
1157 break;
1158
1159 if (!emac_rx_process_rrd(adpt, rx_q, &rrd))
1160 break;
1161
1162 if (likely(RRD_NOR(&rrd) == 1)) {
1163 /* good receive */
1164 rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd));
1165 dma_unmap_single(adpt->netdev->dev.parent,
1166 rfbuf->dma_addr, rfbuf->length,
1167 DMA_FROM_DEVICE);
1168 rfbuf->dma_addr = 0;
1169 skb = rfbuf->skb;
1170 } else {
1171 netdev_err(adpt->netdev,
1172 "error: multi-RFD not support yet!\n");
1173 break;
1174 }
1175 emac_rx_rfd_clean(rx_q, &rrd);
1176 num_consume_pkts--;
1177 count++;
1178
1179 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags
1180 * with DF set are marked as error), drop packets based on the
1181 * error mask rather than the summary bit (ignoring L4F errors)
1182 */
1183 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) {
1184 netif_dbg(adpt, rx_status, adpt->netdev,
1185 "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
1186 rrd.word[0], rrd.word[1],
1187 rrd.word[2], rrd.word[3]);
1188
1189 dev_kfree_skb(skb);
1190 continue;
1191 }
1192
1193 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN);
1194 skb->dev = netdev;
1195 skb->protocol = eth_type_trans(skb, skb->dev);
1196 if (netdev->features & NETIF_F_RXCSUM)
1197 skb->ip_summed = RRD_L4F(&rrd) ?
1198 CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
1199 else
1200 skb_checksum_none_assert(skb);
1201
1202 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd),
1203 (bool)RRD_CVTAG(&rrd));
1204
1205 netdev->last_rx = jiffies;
1206 (*num_pkts)++;
1207 } while (*num_pkts < max_pkts);
1208
1209 if (count) {
1210 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) &
1211 rx_q->process_mask;
1212 emac_reg_update32(adpt->base + rx_q->process_reg,
1213 rx_q->process_mask, proc_idx);
1214 emac_mac_rx_descs_refill(adpt, rx_q);
1215 }
1216 }
1217
1218 /* get the number of free transmit descriptors */
1219 static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q)
1220 {
1221 u32 produce_idx = tx_q->tpd.produce_idx;
1222 u32 consume_idx = tx_q->tpd.consume_idx;
1223
1224 return (consume_idx > produce_idx) ?
1225 (consume_idx - produce_idx - 1) :
1226 (tx_q->tpd.count + consume_idx - produce_idx - 1);
1227 }
1228
1229 /* Process transmit event */
1230 void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)
1231 {
1232 u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg);
1233 u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0;
1234 struct emac_buffer *tpbuf;
1235
1236 hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift;
1237
1238 while (tx_q->tpd.consume_idx != hw_consume_idx) {
1239 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);
1240 if (tpbuf->dma_addr) {
1241 dma_unmap_single(adpt->netdev->dev.parent,
1242 tpbuf->dma_addr, tpbuf->length,
1243 DMA_TO_DEVICE);
1244 tpbuf->dma_addr = 0;
1245 }
1246
1247 if (tpbuf->skb) {
1248 pkts_compl++;
1249 bytes_compl += tpbuf->skb->len;
1250 dev_kfree_skb_irq(tpbuf->skb);
1251 tpbuf->skb = NULL;
1252 }
1253
1254 if (++tx_q->tpd.consume_idx == tx_q->tpd.count)
1255 tx_q->tpd.consume_idx = 0;
1256 }
1257
1258 netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);
1259
1260 if (netif_queue_stopped(adpt->netdev))
1261 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1))
1262 netif_wake_queue(adpt->netdev);
1263 }
1264
1265 /* Initialize all queue data structures */
1266 void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,
1267 struct emac_adapter *adpt)
1268 {
1269 adpt->rx_q.netdev = adpt->netdev;
1270
1271 adpt->rx_q.produce_reg = EMAC_MAILBOX_0;
1272 adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK;
1273 adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT;
1274
1275 adpt->rx_q.process_reg = EMAC_MAILBOX_0;
1276 adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK;
1277 adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT;
1278
1279 adpt->rx_q.consume_reg = EMAC_MAILBOX_3;
1280 adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK;
1281 adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT;
1282
1283 adpt->rx_q.irq = &adpt->irq;
1284 adpt->rx_q.intr = adpt->irq.mask & ISR_RX_PKT;
1285
1286 adpt->tx_q.produce_reg = EMAC_MAILBOX_15;
1287 adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK;
1288 adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT;
1289
1290 adpt->tx_q.consume_reg = EMAC_MAILBOX_2;
1291 adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK;
1292 adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT;
1293 }
1294
1295 /* Fill up transmit descriptors with TSO and Checksum offload information */
1296 static int emac_tso_csum(struct emac_adapter *adpt,
1297 struct emac_tx_queue *tx_q,
1298 struct sk_buff *skb,
1299 struct emac_tpd *tpd)
1300 {
1301 unsigned int hdr_len;
1302 int ret;
1303
1304 if (skb_is_gso(skb)) {
1305 if (skb_header_cloned(skb)) {
1306 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1307 if (unlikely(ret))
1308 return ret;
1309 }
1310
1311 if (skb->protocol == htons(ETH_P_IP)) {
1312 u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data)
1313 + ntohs(ip_hdr(skb)->tot_len);
1314 if (skb->len > pkt_len)
1315 pskb_trim(skb, pkt_len);
1316 }
1317
1318 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1319 if (unlikely(skb->len == hdr_len)) {
1320 /* we only need to do csum */
1321 netif_warn(adpt, tx_err, adpt->netdev,
1322 "tso not needed for packet with 0 data\n");
1323 goto do_csum;
1324 }
1325
1326 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1327 ip_hdr(skb)->check = 0;
1328 tcp_hdr(skb)->check =
1329 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1330 ip_hdr(skb)->daddr,
1331 0, IPPROTO_TCP, 0);
1332 TPD_IPV4_SET(tpd, 1);
1333 }
1334
1335 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
1336 /* ipv6 tso need an extra tpd */
1337 struct emac_tpd extra_tpd;
1338
1339 memset(tpd, 0, sizeof(*tpd));
1340 memset(&extra_tpd, 0, sizeof(extra_tpd));
1341
1342 ipv6_hdr(skb)->payload_len = 0;
1343 tcp_hdr(skb)->check =
1344 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1345 &ipv6_hdr(skb)->daddr,
1346 0, IPPROTO_TCP, 0);
1347 TPD_PKT_LEN_SET(&extra_tpd, skb->len);
1348 TPD_LSO_SET(&extra_tpd, 1);
1349 TPD_LSOV_SET(&extra_tpd, 1);
1350 emac_tx_tpd_create(adpt, tx_q, &extra_tpd);
1351 TPD_LSOV_SET(tpd, 1);
1352 }
1353
1354 TPD_LSO_SET(tpd, 1);
1355 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb));
1356 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size);
1357 return 0;
1358 }
1359
1360 do_csum:
1361 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1362 unsigned int css, cso;
1363
1364 cso = skb_transport_offset(skb);
1365 if (unlikely(cso & 0x1)) {
1366 netdev_err(adpt->netdev,
1367 "error: payload offset should be even\n");
1368 return -EINVAL;
1369 }
1370 css = cso + skb->csum_offset;
1371
1372 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1);
1373 TPD_CXSUM_OFFSET_SET(tpd, css >> 1);
1374 TPD_CSX_SET(tpd, 1);
1375 }
1376
1377 return 0;
1378 }
1379
1380 /* Fill up transmit descriptors */
1381 static void emac_tx_fill_tpd(struct emac_adapter *adpt,
1382 struct emac_tx_queue *tx_q, struct sk_buff *skb,
1383 struct emac_tpd *tpd)
1384 {
1385 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
1386 unsigned int first = tx_q->tpd.produce_idx;
1387 unsigned int len = skb_headlen(skb);
1388 struct emac_buffer *tpbuf = NULL;
1389 unsigned int mapped_len = 0;
1390 unsigned int i;
1391 int count = 0;
1392 int ret;
1393
1394 /* if Large Segment Offload is (in TCP Segmentation Offload struct) */
1395 if (TPD_LSO(tpd)) {
1396 mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1397
1398 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1399 tpbuf->length = mapped_len;
1400 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1401 skb->data, tpbuf->length,
1402 DMA_TO_DEVICE);
1403 ret = dma_mapping_error(adpt->netdev->dev.parent,
1404 tpbuf->dma_addr);
1405 if (ret)
1406 goto error;
1407
1408 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1409 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1410 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1411 emac_tx_tpd_create(adpt, tx_q, tpd);
1412 count++;
1413 }
1414
1415 if (mapped_len < len) {
1416 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1417 tpbuf->length = len - mapped_len;
1418 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1419 skb->data + mapped_len,
1420 tpbuf->length, DMA_TO_DEVICE);
1421 ret = dma_mapping_error(adpt->netdev->dev.parent,
1422 tpbuf->dma_addr);
1423 if (ret)
1424 goto error;
1425
1426 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1427 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1428 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1429 emac_tx_tpd_create(adpt, tx_q, tpd);
1430 count++;
1431 }
1432
1433 for (i = 0; i < nr_frags; i++) {
1434 struct skb_frag_struct *frag;
1435
1436 frag = &skb_shinfo(skb)->frags[i];
1437
1438 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1439 tpbuf->length = frag->size;
1440 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1441 frag->page.p, frag->page_offset,
1442 tpbuf->length, DMA_TO_DEVICE);
1443 ret = dma_mapping_error(adpt->netdev->dev.parent,
1444 tpbuf->dma_addr);
1445 if (ret)
1446 goto error;
1447
1448 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1449 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1450 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1451 emac_tx_tpd_create(adpt, tx_q, tpd);
1452 count++;
1453 }
1454
1455 /* The last tpd */
1456 wmb();
1457 emac_tx_tpd_mark_last(adpt, tx_q);
1458
1459 /* The last buffer info contain the skb address,
1460 * so it will be freed after unmap
1461 */
1462 tpbuf->skb = skb;
1463
1464 return;
1465
1466 error:
1467 /* One of the memory mappings failed, so undo everything */
1468 tx_q->tpd.produce_idx = first;
1469
1470 while (count--) {
1471 tpbuf = GET_TPD_BUFFER(tx_q, first);
1472 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr,
1473 tpbuf->length, DMA_TO_DEVICE);
1474 tpbuf->dma_addr = 0;
1475 tpbuf->length = 0;
1476
1477 if (++first == tx_q->tpd.count)
1478 first = 0;
1479 }
1480
1481 dev_kfree_skb(skb);
1482 }
1483
1484 /* Transmit the packet using specified transmit queue */
1485 int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q,
1486 struct sk_buff *skb)
1487 {
1488 struct emac_tpd tpd;
1489 u32 prod_idx;
1490
1491 memset(&tpd, 0, sizeof(tpd));
1492
1493 if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) {
1494 dev_kfree_skb_any(skb);
1495 return NETDEV_TX_OK;
1496 }
1497
1498 if (skb_vlan_tag_present(skb)) {
1499 u16 tag;
1500
1501 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag);
1502 TPD_CVLAN_TAG_SET(&tpd, tag);
1503 TPD_INSTC_SET(&tpd, 1);
1504 }
1505
1506 if (skb_network_offset(skb) != ETH_HLEN)
1507 TPD_TYP_SET(&tpd, 1);
1508
1509 emac_tx_fill_tpd(adpt, tx_q, skb, &tpd);
1510
1511 netdev_sent_queue(adpt->netdev, skb->len);
1512
1513 /* Make sure the are enough free descriptors to hold one
1514 * maximum-sized SKB. We need one desc for each fragment,
1515 * one for the checksum (emac_tso_csum), one for TSO, and
1516 * and one for the SKB header.
1517 */
1518 if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3))
1519 netif_stop_queue(adpt->netdev);
1520
1521 /* update produce idx */
1522 prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) &
1523 tx_q->produce_mask;
1524 emac_reg_update32(adpt->base + tx_q->produce_reg,
1525 tx_q->produce_mask, prod_idx);
1526
1527 return NETDEV_TX_OK;
1528 }
Linux kernel - Deliverables
This page took 0.063031 seconds and 5 git commands to generate.