2 * Copyright (c) 2012-2014 Qualcomm Atheros, Inc.
4 * Permission to use, copy, modify, and/or distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 #include <linux/etherdevice.h>
18 #include <net/ieee80211_radiotap.h>
19 #include <linux/if_arp.h>
20 #include <linux/moduleparam.h>
22 #include <linux/ipv6.h>
24 #include <linux/prefetch.h>
31 static bool rtap_include_phy_info
;
32 module_param(rtap_include_phy_info
, bool, S_IRUGO
);
33 MODULE_PARM_DESC(rtap_include_phy_info
,
34 " Include PHY info in the radiotap header, default - no");
36 static inline int wil_vring_is_empty(struct vring
*vring
)
38 return vring
->swhead
== vring
->swtail
;
41 static inline u32
wil_vring_next_tail(struct vring
*vring
)
43 return (vring
->swtail
+ 1) % vring
->size
;
46 static inline void wil_vring_advance_head(struct vring
*vring
, int n
)
48 vring
->swhead
= (vring
->swhead
+ n
) % vring
->size
;
51 static inline int wil_vring_is_full(struct vring
*vring
)
53 return wil_vring_next_tail(vring
) == vring
->swhead
;
56 /* Used space in Tx Vring */
57 static inline int wil_vring_used_tx(struct vring
*vring
)
59 u32 swhead
= vring
->swhead
;
60 u32 swtail
= vring
->swtail
;
61 return (vring
->size
+ swhead
- swtail
) % vring
->size
;
64 /* Available space in Tx Vring */
65 static inline int wil_vring_avail_tx(struct vring
*vring
)
67 return vring
->size
- wil_vring_used_tx(vring
) - 1;
70 /* wil_vring_wmark_low - low watermark for available descriptor space */
71 static inline int wil_vring_wmark_low(struct vring
*vring
)
76 /* wil_vring_wmark_high - high watermark for available descriptor space */
77 static inline int wil_vring_wmark_high(struct vring
*vring
)
82 /* wil_val_in_range - check if value in [min,max) */
83 static inline bool wil_val_in_range(int val
, int min
, int max
)
85 return val
>= min
&& val
< max
;
88 static int wil_vring_alloc(struct wil6210_priv
*wil
, struct vring
*vring
)
90 struct device
*dev
= wil_to_dev(wil
);
91 size_t sz
= vring
->size
* sizeof(vring
->va
[0]);
94 wil_dbg_misc(wil
, "%s()\n", __func__
);
96 BUILD_BUG_ON(sizeof(vring
->va
[0]) != 32);
100 vring
->ctx
= kcalloc(vring
->size
, sizeof(vring
->ctx
[0]), GFP_KERNEL
);
105 /* vring->va should be aligned on its size rounded up to power of 2
106 * This is granted by the dma_alloc_coherent
108 vring
->va
= dma_alloc_coherent(dev
, sz
, &vring
->pa
, GFP_KERNEL
);
114 /* initially, all descriptors are SW owned
115 * For Tx and Rx, ownership bit is at the same location, thus
118 for (i
= 0; i
< vring
->size
; i
++) {
119 volatile struct vring_tx_desc
*_d
= &vring
->va
[i
].tx
;
121 _d
->dma
.status
= TX_DMA_STATUS_DU
;
124 wil_dbg_misc(wil
, "vring[%d] 0x%p:%pad 0x%p\n", vring
->size
,
125 vring
->va
, &vring
->pa
, vring
->ctx
);
130 static void wil_txdesc_unmap(struct device
*dev
, struct vring_tx_desc
*d
,
133 dma_addr_t pa
= wil_desc_addr(&d
->dma
.addr
);
134 u16 dmalen
= le16_to_cpu(d
->dma
.length
);
136 switch (ctx
->mapped_as
) {
137 case wil_mapped_as_single
:
138 dma_unmap_single(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
140 case wil_mapped_as_page
:
141 dma_unmap_page(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
148 static void wil_vring_free(struct wil6210_priv
*wil
, struct vring
*vring
,
151 struct device
*dev
= wil_to_dev(wil
);
152 size_t sz
= vring
->size
* sizeof(vring
->va
[0]);
155 int vring_index
= vring
- wil
->vring_tx
;
157 wil_dbg_misc(wil
, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n",
158 vring_index
, vring
->size
, vring
->va
,
159 &vring
->pa
, vring
->ctx
);
161 wil_dbg_misc(wil
, "free Rx vring [%d] 0x%p:%pad 0x%p\n",
162 vring
->size
, vring
->va
,
163 &vring
->pa
, vring
->ctx
);
166 while (!wil_vring_is_empty(vring
)) {
172 struct vring_tx_desc dd
, *d
= &dd
;
173 volatile struct vring_tx_desc
*_d
=
174 &vring
->va
[vring
->swtail
].tx
;
176 ctx
= &vring
->ctx
[vring
->swtail
];
178 wil_txdesc_unmap(dev
, d
, ctx
);
180 dev_kfree_skb_any(ctx
->skb
);
181 vring
->swtail
= wil_vring_next_tail(vring
);
183 struct vring_rx_desc dd
, *d
= &dd
;
184 volatile struct vring_rx_desc
*_d
=
185 &vring
->va
[vring
->swhead
].rx
;
187 ctx
= &vring
->ctx
[vring
->swhead
];
189 pa
= wil_desc_addr(&d
->dma
.addr
);
190 dmalen
= le16_to_cpu(d
->dma
.length
);
191 dma_unmap_single(dev
, pa
, dmalen
, DMA_FROM_DEVICE
);
193 wil_vring_advance_head(vring
, 1);
196 dma_free_coherent(dev
, sz
, (void *)vring
->va
, vring
->pa
);
204 * Allocate one skb for Rx VRING
206 * Safe to call from IRQ
208 static int wil_vring_alloc_skb(struct wil6210_priv
*wil
, struct vring
*vring
,
211 struct device
*dev
= wil_to_dev(wil
);
212 unsigned int sz
= mtu_max
+ ETH_HLEN
;
213 struct vring_rx_desc dd
, *d
= &dd
;
214 volatile struct vring_rx_desc
*_d
= &vring
->va
[i
].rx
;
216 struct sk_buff
*skb
= dev_alloc_skb(sz
+ headroom
);
221 skb_reserve(skb
, headroom
);
224 pa
= dma_map_single(dev
, skb
->data
, skb
->len
, DMA_FROM_DEVICE
);
225 if (unlikely(dma_mapping_error(dev
, pa
))) {
230 d
->dma
.d0
= BIT(9) | RX_DMA_D0_CMD_DMA_IT
;
231 wil_desc_addr_set(&d
->dma
.addr
, pa
);
232 /* ip_length don't care */
234 /* error don't care */
235 d
->dma
.status
= 0; /* BIT(0) should be 0 for HW_OWNED */
236 d
->dma
.length
= cpu_to_le16(sz
);
238 vring
->ctx
[i
].skb
= skb
;
244 * Adds radiotap header
246 * Any error indicated as "Bad FCS"
248 * Vendor data for 04:ce:14-1 (Wilocity-1) consists of:
249 * - Rx descriptor: 32 bytes
252 static void wil_rx_add_radiotap_header(struct wil6210_priv
*wil
,
255 struct wireless_dev
*wdev
= wil
->wdev
;
256 struct wil6210_rtap
{
257 struct ieee80211_radiotap_header rthdr
;
258 /* fields should be in the order of bits in rthdr.it_present */
262 __le16 chnl_freq
__aligned(2);
269 struct wil6210_rtap_vendor
{
270 struct wil6210_rtap rtap
;
272 u8 vendor_oui
[3] __aligned(2);
277 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
278 struct wil6210_rtap_vendor
*rtap_vendor
;
279 int rtap_len
= sizeof(struct wil6210_rtap
);
280 int phy_length
= 0; /* phy info header size, bytes */
281 static char phy_data
[128];
282 struct ieee80211_channel
*ch
= wdev
->preset_chandef
.chan
;
284 if (rtap_include_phy_info
) {
285 rtap_len
= sizeof(*rtap_vendor
) + sizeof(*d
);
286 /* calculate additional length */
287 if (d
->dma
.status
& RX_DMA_STATUS_PHY_INFO
) {
289 * PHY info starts from 8-byte boundary
290 * there are 8-byte lines, last line may be partially
291 * written (HW bug), thus FW configures for last line
292 * to be excessive. Driver skips this last line.
294 int len
= min_t(int, 8 + sizeof(phy_data
),
295 wil_rxdesc_phy_length(d
));
298 void *p
= skb_tail_pointer(skb
);
299 void *pa
= PTR_ALIGN(p
, 8);
301 if (skb_tailroom(skb
) >= len
+ (pa
- p
)) {
302 phy_length
= len
- 8;
303 memcpy(phy_data
, pa
, phy_length
);
307 rtap_len
+= phy_length
;
310 if (skb_headroom(skb
) < rtap_len
&&
311 pskb_expand_head(skb
, rtap_len
, 0, GFP_ATOMIC
)) {
312 wil_err(wil
, "Unable to expand headrom to %d\n", rtap_len
);
316 rtap_vendor
= (void *)skb_push(skb
, rtap_len
);
317 memset(rtap_vendor
, 0, rtap_len
);
319 rtap_vendor
->rtap
.rthdr
.it_version
= PKTHDR_RADIOTAP_VERSION
;
320 rtap_vendor
->rtap
.rthdr
.it_len
= cpu_to_le16(rtap_len
);
321 rtap_vendor
->rtap
.rthdr
.it_present
= cpu_to_le32(
322 (1 << IEEE80211_RADIOTAP_FLAGS
) |
323 (1 << IEEE80211_RADIOTAP_CHANNEL
) |
324 (1 << IEEE80211_RADIOTAP_MCS
));
325 if (d
->dma
.status
& RX_DMA_STATUS_ERROR
)
326 rtap_vendor
->rtap
.flags
|= IEEE80211_RADIOTAP_F_BADFCS
;
328 rtap_vendor
->rtap
.chnl_freq
= cpu_to_le16(ch
? ch
->center_freq
: 58320);
329 rtap_vendor
->rtap
.chnl_flags
= cpu_to_le16(0);
331 rtap_vendor
->rtap
.mcs_present
= IEEE80211_RADIOTAP_MCS_HAVE_MCS
;
332 rtap_vendor
->rtap
.mcs_flags
= 0;
333 rtap_vendor
->rtap
.mcs_index
= wil_rxdesc_mcs(d
);
335 if (rtap_include_phy_info
) {
336 rtap_vendor
->rtap
.rthdr
.it_present
|= cpu_to_le32(1 <<
337 IEEE80211_RADIOTAP_VENDOR_NAMESPACE
);
338 /* OUI for Wilocity 04:ce:14 */
339 rtap_vendor
->vendor_oui
[0] = 0x04;
340 rtap_vendor
->vendor_oui
[1] = 0xce;
341 rtap_vendor
->vendor_oui
[2] = 0x14;
342 rtap_vendor
->vendor_ns
= 1;
343 /* Rx descriptor + PHY data */
344 rtap_vendor
->vendor_skip
= cpu_to_le16(sizeof(*d
) +
346 memcpy(rtap_vendor
->vendor_data
, (void *)d
, sizeof(*d
));
347 memcpy(rtap_vendor
->vendor_data
+ sizeof(*d
), phy_data
,
353 * reap 1 frame from @swhead
355 * Rx descriptor copied to skb->cb
357 * Safe to call from IRQ
359 static struct sk_buff
*wil_vring_reap_rx(struct wil6210_priv
*wil
,
362 struct device
*dev
= wil_to_dev(wil
);
363 struct net_device
*ndev
= wil_to_ndev(wil
);
364 volatile struct vring_rx_desc
*_d
;
365 struct vring_rx_desc
*d
;
368 unsigned int sz
= mtu_max
+ ETH_HLEN
;
372 struct wil_net_stats
*stats
;
374 BUILD_BUG_ON(sizeof(struct vring_rx_desc
) > sizeof(skb
->cb
));
376 if (unlikely(wil_vring_is_empty(vring
)))
379 _d
= &vring
->va
[vring
->swhead
].rx
;
380 if (unlikely(!(_d
->dma
.status
& RX_DMA_STATUS_DU
))) {
381 /* it is not error, we just reached end of Rx done area */
385 skb
= vring
->ctx
[vring
->swhead
].skb
;
386 d
= wil_skb_rxdesc(skb
);
388 pa
= wil_desc_addr(&d
->dma
.addr
);
389 vring
->ctx
[vring
->swhead
].skb
= NULL
;
390 wil_vring_advance_head(vring
, 1);
392 dma_unmap_single(dev
, pa
, sz
, DMA_FROM_DEVICE
);
393 dmalen
= le16_to_cpu(d
->dma
.length
);
395 trace_wil6210_rx(vring
->swhead
, d
);
396 wil_dbg_txrx(wil
, "Rx[%3d] : %d bytes\n", vring
->swhead
, dmalen
);
397 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_NONE
, 32, 4,
398 (const void *)d
, sizeof(*d
), false);
400 if (unlikely(dmalen
> sz
)) {
401 wil_err(wil
, "Rx size too large: %d bytes!\n", dmalen
);
405 skb_trim(skb
, dmalen
);
409 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET
, 16, 1,
410 skb
->data
, skb_headlen(skb
), false);
412 cid
= wil_rxdesc_cid(d
);
413 stats
= &wil
->sta
[cid
].stats
;
414 stats
->last_mcs_rx
= wil_rxdesc_mcs(d
);
416 /* use radiotap header only if required */
417 if (ndev
->type
== ARPHRD_IEEE80211_RADIOTAP
)
418 wil_rx_add_radiotap_header(wil
, skb
);
420 /* no extra checks if in sniffer mode */
421 if (ndev
->type
!= ARPHRD_ETHER
)
424 * Non-data frames may be delivered through Rx DMA channel (ex: BAR)
425 * Driver should recognize it by frame type, that is found
426 * in Rx descriptor. If type is not data, it is 802.11 frame as is
428 ftype
= wil_rxdesc_ftype(d
) << 2;
429 if (unlikely(ftype
!= IEEE80211_FTYPE_DATA
)) {
430 wil_dbg_txrx(wil
, "Non-data frame ftype 0x%08x\n", ftype
);
431 /* TODO: process it */
436 if (unlikely(skb
->len
< ETH_HLEN
)) {
437 wil_err(wil
, "Short frame, len = %d\n", skb
->len
);
438 /* TODO: process it (i.e. BAR) */
443 /* L4 IDENT is on when HW calculated checksum, check status
444 * and in case of error drop the packet
445 * higher stack layers will handle retransmission (if required)
447 if (likely(d
->dma
.status
& RX_DMA_STATUS_L4I
)) {
448 /* L4 protocol identified, csum calculated */
449 if (likely((d
->dma
.error
& RX_DMA_ERROR_L4_ERR
) == 0))
450 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
451 /* If HW reports bad checksum, let IP stack re-check it
452 * For example, HW don't understand Microsoft IP stack that
453 * mis-calculates TCP checksum - if it should be 0x0,
454 * it writes 0xffff in violation of RFC 1624
462 * allocate and fill up to @count buffers in rx ring
463 * buffers posted at @swtail
465 static int wil_rx_refill(struct wil6210_priv
*wil
, int count
)
467 struct net_device
*ndev
= wil_to_ndev(wil
);
468 struct vring
*v
= &wil
->vring_rx
;
471 int headroom
= ndev
->type
== ARPHRD_IEEE80211_RADIOTAP
?
472 WIL6210_RTAP_SIZE
: 0;
474 for (; next_tail
= wil_vring_next_tail(v
),
475 (next_tail
!= v
->swhead
) && (count
-- > 0);
476 v
->swtail
= next_tail
) {
477 rc
= wil_vring_alloc_skb(wil
, v
, v
->swtail
, headroom
);
479 wil_err(wil
, "Error %d in wil_rx_refill[%d]\n",
484 iowrite32(v
->swtail
, wil
->csr
+ HOSTADDR(v
->hwtail
));
490 * Pass Rx packet to the netif. Update statistics.
491 * Called in softirq context (NAPI poll).
493 void wil_netif_rx_any(struct sk_buff
*skb
, struct net_device
*ndev
)
495 gro_result_t rc
= GRO_NORMAL
;
496 struct wil6210_priv
*wil
= ndev_to_wil(ndev
);
497 struct wireless_dev
*wdev
= wil_to_wdev(wil
);
498 unsigned int len
= skb
->len
;
499 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
500 int cid
= wil_rxdesc_cid(d
);
501 struct ethhdr
*eth
= (void *)skb
->data
;
502 /* here looking for DA, not A1, thus Rxdesc's 'mcast' indication
503 * is not suitable, need to look at data
505 int mcast
= is_multicast_ether_addr(eth
->h_dest
);
506 struct wil_net_stats
*stats
= &wil
->sta
[cid
].stats
;
507 struct sk_buff
*xmit_skb
= NULL
;
508 static const char * const gro_res_str
[] = {
509 [GRO_MERGED
] = "GRO_MERGED",
510 [GRO_MERGED_FREE
] = "GRO_MERGED_FREE",
511 [GRO_HELD
] = "GRO_HELD",
512 [GRO_NORMAL
] = "GRO_NORMAL",
513 [GRO_DROP
] = "GRO_DROP",
518 if (wdev
->iftype
== NL80211_IFTYPE_AP
&& !wil
->ap_isolate
) {
520 /* send multicast frames both to higher layers in
521 * local net stack and back to the wireless medium
523 xmit_skb
= skb_copy(skb
, GFP_ATOMIC
);
525 int xmit_cid
= wil_find_cid(wil
, eth
->h_dest
);
528 /* The destination station is associated to
529 * this AP (in this VLAN), so send the frame
530 * directly to it and do not pass it to local
539 /* Send to wireless media and increase priority by 256 to
540 * keep the received priority instead of reclassifying
541 * the frame (see cfg80211_classify8021d).
543 xmit_skb
->dev
= ndev
;
544 xmit_skb
->priority
+= 256;
545 xmit_skb
->protocol
= htons(ETH_P_802_3
);
546 skb_reset_network_header(xmit_skb
);
547 skb_reset_mac_header(xmit_skb
);
548 wil_dbg_txrx(wil
, "Rx -> Tx %d bytes\n", len
);
549 dev_queue_xmit(xmit_skb
);
552 if (skb
) { /* deliver to local stack */
554 skb
->protocol
= eth_type_trans(skb
, ndev
);
555 rc
= napi_gro_receive(&wil
->napi_rx
, skb
);
556 wil_dbg_txrx(wil
, "Rx complete %d bytes => %s\n",
557 len
, gro_res_str
[rc
]);
559 /* statistics. rc set to GRO_NORMAL for AP bridging */
560 if (unlikely(rc
== GRO_DROP
)) {
561 ndev
->stats
.rx_dropped
++;
563 wil_dbg_txrx(wil
, "Rx drop %d bytes\n", len
);
565 ndev
->stats
.rx_packets
++;
567 ndev
->stats
.rx_bytes
+= len
;
568 stats
->rx_bytes
+= len
;
570 ndev
->stats
.multicast
++;
575 * Proceed all completed skb's from Rx VRING
577 * Safe to call from NAPI poll, i.e. softirq with interrupts enabled
579 void wil_rx_handle(struct wil6210_priv
*wil
, int *quota
)
581 struct net_device
*ndev
= wil_to_ndev(wil
);
582 struct vring
*v
= &wil
->vring_rx
;
585 if (unlikely(!v
->va
)) {
586 wil_err(wil
, "Rx IRQ while Rx not yet initialized\n");
589 wil_dbg_txrx(wil
, "%s()\n", __func__
);
590 while ((*quota
> 0) && (NULL
!= (skb
= wil_vring_reap_rx(wil
, v
)))) {
593 if (wil
->wdev
->iftype
== NL80211_IFTYPE_MONITOR
) {
595 skb_reset_mac_header(skb
);
596 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
597 skb
->pkt_type
= PACKET_OTHERHOST
;
598 skb
->protocol
= htons(ETH_P_802_2
);
599 wil_netif_rx_any(skb
, ndev
);
601 wil_rx_reorder(wil
, skb
);
604 wil_rx_refill(wil
, v
->size
);
607 int wil_rx_init(struct wil6210_priv
*wil
, u16 size
)
609 struct vring
*vring
= &wil
->vring_rx
;
612 wil_dbg_misc(wil
, "%s()\n", __func__
);
615 wil_err(wil
, "Rx ring already allocated\n");
620 rc
= wil_vring_alloc(wil
, vring
);
624 rc
= wmi_rx_chain_add(wil
, vring
);
628 rc
= wil_rx_refill(wil
, vring
->size
);
634 wil_vring_free(wil
, vring
, 0);
639 void wil_rx_fini(struct wil6210_priv
*wil
)
641 struct vring
*vring
= &wil
->vring_rx
;
643 wil_dbg_misc(wil
, "%s()\n", __func__
);
646 wil_vring_free(wil
, vring
, 0);
649 int wil_vring_init_tx(struct wil6210_priv
*wil
, int id
, int size
,
653 struct wmi_vring_cfg_cmd cmd
= {
654 .action
= cpu_to_le32(WMI_VRING_CMD_ADD
),
658 cpu_to_le16(wil_mtu2macbuf(mtu_max
)),
659 .ring_size
= cpu_to_le16(size
),
662 .cidxtid
= mk_cidxtid(cid
, tid
),
663 .encap_trans_type
= WMI_VRING_ENC_TYPE_802_3
,
668 .priority
= cpu_to_le16(0),
669 .timeslot_us
= cpu_to_le16(0xfff),
674 struct wil6210_mbox_hdr_wmi wmi
;
675 struct wmi_vring_cfg_done_event cmd
;
677 struct vring
*vring
= &wil
->vring_tx
[id
];
678 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[id
];
680 wil_dbg_misc(wil
, "%s() max_mpdu_size %d\n", __func__
,
681 cmd
.vring_cfg
.tx_sw_ring
.max_mpdu_size
);
684 wil_err(wil
, "Tx ring [%d] already allocated\n", id
);
689 memset(txdata
, 0, sizeof(*txdata
));
690 spin_lock_init(&txdata
->lock
);
692 rc
= wil_vring_alloc(wil
, vring
);
696 wil
->vring2cid_tid
[id
][0] = cid
;
697 wil
->vring2cid_tid
[id
][1] = tid
;
699 cmd
.vring_cfg
.tx_sw_ring
.ring_mem_base
= cpu_to_le64(vring
->pa
);
701 rc
= wmi_call(wil
, WMI_VRING_CFG_CMDID
, &cmd
, sizeof(cmd
),
702 WMI_VRING_CFG_DONE_EVENTID
, &reply
, sizeof(reply
), 100);
706 if (reply
.cmd
.status
!= WMI_FW_STATUS_SUCCESS
) {
707 wil_err(wil
, "Tx config failed, status 0x%02x\n",
712 vring
->hwtail
= le32_to_cpu(reply
.cmd
.tx_vring_tail_ptr
);
715 if (wil
->sta
[cid
].data_port_open
&& (agg_wsize
>= 0))
716 wil_addba_tx_request(wil
, id
, agg_wsize
);
720 wil_vring_free(wil
, vring
, 1);
726 void wil_vring_fini_tx(struct wil6210_priv
*wil
, int id
)
728 struct vring
*vring
= &wil
->vring_tx
[id
];
729 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[id
];
731 WARN_ON(!mutex_is_locked(&wil
->mutex
));
736 wil_dbg_misc(wil
, "%s() id=%d\n", __func__
, id
);
738 spin_lock_bh(&txdata
->lock
);
739 txdata
->enabled
= 0; /* no Tx can be in progress or start anew */
740 spin_unlock_bh(&txdata
->lock
);
741 /* make sure NAPI won't touch this vring */
742 if (test_bit(wil_status_napi_en
, wil
->status
))
743 napi_synchronize(&wil
->napi_tx
);
745 wil_vring_free(wil
, vring
, 1);
746 memset(txdata
, 0, sizeof(*txdata
));
749 static struct vring
*wil_find_tx_vring(struct wil6210_priv
*wil
,
753 struct ethhdr
*eth
= (void *)skb
->data
;
754 int cid
= wil_find_cid(wil
, eth
->h_dest
);
759 if (!wil
->sta
[cid
].data_port_open
&&
760 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
763 /* TODO: fix for multiple TID */
764 for (i
= 0; i
< ARRAY_SIZE(wil
->vring2cid_tid
); i
++) {
765 if (wil
->vring2cid_tid
[i
][0] == cid
) {
766 struct vring
*v
= &wil
->vring_tx
[i
];
768 wil_dbg_txrx(wil
, "%s(%pM) -> [%d]\n",
769 __func__
, eth
->h_dest
, i
);
773 wil_dbg_txrx(wil
, "vring[%d] not valid\n", i
);
782 static void wil_set_da_for_vring(struct wil6210_priv
*wil
,
783 struct sk_buff
*skb
, int vring_index
)
785 struct ethhdr
*eth
= (void *)skb
->data
;
786 int cid
= wil
->vring2cid_tid
[vring_index
][0];
788 memcpy(eth
->h_dest
, wil
->sta
[cid
].addr
, ETH_ALEN
);
791 static int wil_tx_vring(struct wil6210_priv
*wil
, struct vring
*vring
,
792 struct sk_buff
*skb
);
794 static struct vring
*wil_find_tx_vring_sta(struct wil6210_priv
*wil
,
801 /* In the STA mode, it is expected to have only 1 VRING
802 * for the AP we connected to.
803 * find 1-st vring and see whether it is eligible for data
805 for (i
= 0; i
< WIL6210_MAX_TX_RINGS
; i
++) {
806 v
= &wil
->vring_tx
[i
];
810 cid
= wil
->vring2cid_tid
[i
][0];
811 if (!wil
->sta
[cid
].data_port_open
&&
812 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
815 wil_dbg_txrx(wil
, "Tx -> ring %d\n", i
);
820 wil_dbg_txrx(wil
, "Tx while no vrings active?\n");
826 * Find 1-st vring and return it; set dest address for this vring in skb
827 * duplicate skb and send it to other active vrings
829 static struct vring
*wil_tx_bcast(struct wil6210_priv
*wil
,
832 struct vring
*v
, *v2
;
833 struct sk_buff
*skb2
;
837 /* find 1-st vring eligible for data */
838 for (i
= 0; i
< WIL6210_MAX_TX_RINGS
; i
++) {
839 v
= &wil
->vring_tx
[i
];
843 cid
= wil
->vring2cid_tid
[i
][0];
844 if (!wil
->sta
[cid
].data_port_open
)
850 wil_dbg_txrx(wil
, "Tx while no vrings active?\n");
855 wil_dbg_txrx(wil
, "BCAST -> ring %d\n", i
);
856 wil_set_da_for_vring(wil
, skb
, i
);
858 /* find other active vrings and duplicate skb for each */
859 for (i
++; i
< WIL6210_MAX_TX_RINGS
; i
++) {
860 v2
= &wil
->vring_tx
[i
];
863 cid
= wil
->vring2cid_tid
[i
][0];
864 if (!wil
->sta
[cid
].data_port_open
)
867 skb2
= skb_copy(skb
, GFP_ATOMIC
);
869 wil_dbg_txrx(wil
, "BCAST DUP -> ring %d\n", i
);
870 wil_set_da_for_vring(wil
, skb2
, i
);
871 wil_tx_vring(wil
, v2
, skb2
);
873 wil_err(wil
, "skb_copy failed\n");
880 static int wil_tx_desc_map(struct vring_tx_desc
*d
, dma_addr_t pa
, u32 len
,
883 wil_desc_addr_set(&d
->dma
.addr
, pa
);
884 d
->dma
.ip_length
= 0;
885 /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/
886 d
->dma
.b11
= 0/*14 | BIT(7)*/;
888 d
->dma
.status
= 0; /* BIT(0) should be 0 for HW_OWNED */
889 d
->dma
.length
= cpu_to_le16((u16
)len
);
890 d
->dma
.d0
= (vring_index
<< DMA_CFG_DESC_TX_0_QID_POS
);
894 d
->mac
.ucode_cmd
= 0;
895 /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi */
896 d
->mac
.d
[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS
) |
897 (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS
);
903 void wil_tx_desc_set_nr_frags(struct vring_tx_desc
*d
, int nr_frags
)
905 d
->mac
.d
[2] |= ((nr_frags
+ 1) <<
906 MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS
);
909 static int wil_tx_desc_offload_cksum_set(struct wil6210_priv
*wil
,
910 struct vring_tx_desc
*d
,
915 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
918 d
->dma
.b11
= ETH_HLEN
; /* MAC header length */
920 switch (skb
->protocol
) {
921 case cpu_to_be16(ETH_P_IP
):
922 protocol
= ip_hdr(skb
)->protocol
;
923 d
->dma
.b11
|= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS
);
925 case cpu_to_be16(ETH_P_IPV6
):
926 protocol
= ipv6_hdr(skb
)->nexthdr
;
934 d
->dma
.d0
|= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS
);
935 /* L4 header len: TCP header length */
937 (tcp_hdrlen(skb
) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
940 /* L4 header len: UDP header length */
942 (sizeof(struct udphdr
) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
948 d
->dma
.ip_length
= skb_network_header_len(skb
);
949 /* Enable TCP/UDP checksum */
950 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS
);
951 /* Calculate pseudo-header */
952 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS
);
957 static int __wil_tx_vring(struct wil6210_priv
*wil
, struct vring
*vring
,
960 struct device
*dev
= wil_to_dev(wil
);
961 struct vring_tx_desc dd
, *d
= &dd
;
962 volatile struct vring_tx_desc
*_d
;
963 u32 swhead
= vring
->swhead
;
964 int avail
= wil_vring_avail_tx(vring
);
965 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
967 int vring_index
= vring
- wil
->vring_tx
;
968 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[vring_index
];
973 wil_dbg_txrx(wil
, "%s()\n", __func__
);
975 if (unlikely(!txdata
->enabled
))
978 if (unlikely(avail
< 1 + nr_frags
)) {
979 wil_err_ratelimited(wil
,
980 "Tx ring[%2d] full. No space for %d fragments\n",
981 vring_index
, 1 + nr_frags
);
984 _d
= &vring
->va
[i
].tx
;
986 pa
= dma_map_single(dev
, skb
->data
, skb_headlen(skb
), DMA_TO_DEVICE
);
988 wil_dbg_txrx(wil
, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", vring_index
,
989 skb_headlen(skb
), skb
->data
, &pa
);
990 wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET
, 16, 1,
991 skb
->data
, skb_headlen(skb
), false);
993 if (unlikely(dma_mapping_error(dev
, pa
)))
995 vring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
997 wil_tx_desc_map(d
, pa
, skb_headlen(skb
), vring_index
);
998 /* Process TCP/UDP checksum offloading */
999 if (unlikely(wil_tx_desc_offload_cksum_set(wil
, d
, skb
))) {
1000 wil_err(wil
, "Tx[%2d] Failed to set cksum, drop packet\n",
1005 vring
->ctx
[i
].nr_frags
= nr_frags
;
1006 wil_tx_desc_set_nr_frags(d
, nr_frags
);
1008 /* middle segments */
1009 for (; f
< nr_frags
; f
++) {
1010 const struct skb_frag_struct
*frag
=
1011 &skb_shinfo(skb
)->frags
[f
];
1012 int len
= skb_frag_size(frag
);
1015 wil_dbg_txrx(wil
, "Tx[%2d] desc[%4d]\n", vring_index
, i
);
1016 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE
, 32, 4,
1017 (const void *)d
, sizeof(*d
), false);
1018 i
= (swhead
+ f
+ 1) % vring
->size
;
1019 _d
= &vring
->va
[i
].tx
;
1020 pa
= skb_frag_dma_map(dev
, frag
, 0, skb_frag_size(frag
),
1022 if (unlikely(dma_mapping_error(dev
, pa
)))
1024 vring
->ctx
[i
].mapped_as
= wil_mapped_as_page
;
1025 wil_tx_desc_map(d
, pa
, len
, vring_index
);
1026 /* no need to check return code -
1027 * if it succeeded for 1-st descriptor,
1028 * it will succeed here too
1030 wil_tx_desc_offload_cksum_set(wil
, d
, skb
);
1032 /* for the last seg only */
1033 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS
);
1034 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS
);
1035 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS
);
1037 wil_dbg_txrx(wil
, "Tx[%2d] desc[%4d]\n", vring_index
, i
);
1038 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE
, 32, 4,
1039 (const void *)d
, sizeof(*d
), false);
1041 /* hold reference to skb
1042 * to prevent skb release before accounting
1043 * in case of immediate "tx done"
1045 vring
->ctx
[i
].skb
= skb_get(skb
);
1047 /* performance monitoring */
1048 used
= wil_vring_used_tx(vring
);
1049 if (wil_val_in_range(vring_idle_trsh
,
1050 used
, used
+ nr_frags
+ 1)) {
1051 txdata
->idle
+= get_cycles() - txdata
->last_idle
;
1052 wil_dbg_txrx(wil
, "Ring[%2d] not idle %d -> %d\n",
1053 vring_index
, used
, used
+ nr_frags
+ 1);
1056 /* advance swhead */
1057 wil_vring_advance_head(vring
, nr_frags
+ 1);
1058 wil_dbg_txrx(wil
, "Tx[%2d] swhead %d -> %d\n", vring_index
, swhead
,
1060 trace_wil6210_tx(vring_index
, swhead
, skb
->len
, nr_frags
);
1061 iowrite32(vring
->swhead
, wil
->csr
+ HOSTADDR(vring
->hwtail
));
1065 /* unmap what we have mapped */
1066 nr_frags
= f
+ 1; /* frags mapped + one for skb head */
1067 for (f
= 0; f
< nr_frags
; f
++) {
1068 struct wil_ctx
*ctx
;
1070 i
= (swhead
+ f
) % vring
->size
;
1071 ctx
= &vring
->ctx
[i
];
1072 _d
= &vring
->va
[i
].tx
;
1074 _d
->dma
.status
= TX_DMA_STATUS_DU
;
1075 wil_txdesc_unmap(dev
, d
, ctx
);
1078 dev_kfree_skb_any(ctx
->skb
);
1080 memset(ctx
, 0, sizeof(*ctx
));
1086 static int wil_tx_vring(struct wil6210_priv
*wil
, struct vring
*vring
,
1087 struct sk_buff
*skb
)
1089 int vring_index
= vring
- wil
->vring_tx
;
1090 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[vring_index
];
1093 spin_lock(&txdata
->lock
);
1094 rc
= __wil_tx_vring(wil
, vring
, skb
);
1095 spin_unlock(&txdata
->lock
);
1099 netdev_tx_t
wil_start_xmit(struct sk_buff
*skb
, struct net_device
*ndev
)
1101 struct wil6210_priv
*wil
= ndev_to_wil(ndev
);
1102 struct ethhdr
*eth
= (void *)skb
->data
;
1103 struct vring
*vring
;
1104 static bool pr_once_fw
;
1107 wil_dbg_txrx(wil
, "%s()\n", __func__
);
1108 if (unlikely(!test_bit(wil_status_fwready
, wil
->status
))) {
1110 wil_err(wil
, "FW not ready\n");
1115 if (unlikely(!test_bit(wil_status_fwconnected
, wil
->status
))) {
1116 wil_err(wil
, "FW not connected\n");
1119 if (unlikely(wil
->wdev
->iftype
== NL80211_IFTYPE_MONITOR
)) {
1120 wil_err(wil
, "Xmit in monitor mode not supported\n");
1126 if (wil
->wdev
->iftype
== NL80211_IFTYPE_STATION
) {
1127 /* in STA mode (ESS), all to same VRING */
1128 vring
= wil_find_tx_vring_sta(wil
, skb
);
1129 } else { /* direct communication, find matching VRING */
1130 if (is_unicast_ether_addr(eth
->h_dest
))
1131 vring
= wil_find_tx_vring(wil
, skb
);
1133 vring
= wil_tx_bcast(wil
, skb
);
1135 if (unlikely(!vring
)) {
1136 wil_dbg_txrx(wil
, "No Tx VRING found for %pM\n", eth
->h_dest
);
1139 /* set up vring entry */
1140 rc
= wil_tx_vring(wil
, vring
, skb
);
1142 /* do we still have enough room in the vring? */
1143 if (unlikely(wil_vring_avail_tx(vring
) < wil_vring_wmark_low(vring
))) {
1144 netif_tx_stop_all_queues(wil_to_ndev(wil
));
1145 wil_dbg_txrx(wil
, "netif_tx_stop : ring full\n");
1150 /* statistics will be updated on the tx_complete */
1151 dev_kfree_skb_any(skb
);
1152 return NETDEV_TX_OK
;
1154 return NETDEV_TX_BUSY
;
1156 break; /* goto drop; */
1159 ndev
->stats
.tx_dropped
++;
1160 dev_kfree_skb_any(skb
);
1162 return NET_XMIT_DROP
;
1165 static inline bool wil_need_txstat(struct sk_buff
*skb
)
1167 struct ethhdr
*eth
= (void *)skb
->data
;
1169 return is_unicast_ether_addr(eth
->h_dest
) && skb
->sk
&&
1170 (skb_shinfo(skb
)->tx_flags
& SKBTX_WIFI_STATUS
);
1173 static inline void wil_consume_skb(struct sk_buff
*skb
, bool acked
)
1175 if (unlikely(wil_need_txstat(skb
)))
1176 skb_complete_wifi_ack(skb
, acked
);
1178 acked
? dev_consume_skb_any(skb
) : dev_kfree_skb_any(skb
);
1182 * Clean up transmitted skb's from the Tx VRING
1184 * Return number of descriptors cleared
1186 * Safe to call from IRQ
1188 int wil_tx_complete(struct wil6210_priv
*wil
, int ringid
)
1190 struct net_device
*ndev
= wil_to_ndev(wil
);
1191 struct device
*dev
= wil_to_dev(wil
);
1192 struct vring
*vring
= &wil
->vring_tx
[ringid
];
1193 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[ringid
];
1195 int cid
= wil
->vring2cid_tid
[ringid
][0];
1196 struct wil_net_stats
*stats
= &wil
->sta
[cid
].stats
;
1197 volatile struct vring_tx_desc
*_d
;
1198 int used_before_complete
;
1201 if (unlikely(!vring
->va
)) {
1202 wil_err(wil
, "Tx irq[%d]: vring not initialized\n", ringid
);
1206 if (unlikely(!txdata
->enabled
)) {
1207 wil_info(wil
, "Tx irq[%d]: vring disabled\n", ringid
);
1211 wil_dbg_txrx(wil
, "%s(%d)\n", __func__
, ringid
);
1213 used_before_complete
= wil_vring_used_tx(vring
);
1215 while (!wil_vring_is_empty(vring
)) {
1217 struct wil_ctx
*ctx
= &vring
->ctx
[vring
->swtail
];
1219 * For the fragmented skb, HW will set DU bit only for the
1220 * last fragment. look for it
1222 int lf
= (vring
->swtail
+ ctx
->nr_frags
) % vring
->size
;
1223 /* TODO: check we are not past head */
1225 _d
= &vring
->va
[lf
].tx
;
1226 if (unlikely(!(_d
->dma
.status
& TX_DMA_STATUS_DU
)))
1229 new_swtail
= (lf
+ 1) % vring
->size
;
1230 while (vring
->swtail
!= new_swtail
) {
1231 struct vring_tx_desc dd
, *d
= &dd
;
1233 struct sk_buff
*skb
;
1235 ctx
= &vring
->ctx
[vring
->swtail
];
1237 _d
= &vring
->va
[vring
->swtail
].tx
;
1241 dmalen
= le16_to_cpu(d
->dma
.length
);
1242 trace_wil6210_tx_done(ringid
, vring
->swtail
, dmalen
,
1245 "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n",
1246 ringid
, vring
->swtail
, dmalen
,
1247 d
->dma
.status
, d
->dma
.error
);
1248 wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE
, 32, 4,
1249 (const void *)d
, sizeof(*d
), false);
1251 wil_txdesc_unmap(dev
, d
, ctx
);
1254 if (likely(d
->dma
.error
== 0)) {
1255 ndev
->stats
.tx_packets
++;
1256 stats
->tx_packets
++;
1257 ndev
->stats
.tx_bytes
+= skb
->len
;
1258 stats
->tx_bytes
+= skb
->len
;
1260 ndev
->stats
.tx_errors
++;
1263 wil_consume_skb(skb
, d
->dma
.error
== 0);
1265 memset(ctx
, 0, sizeof(*ctx
));
1266 /* There is no need to touch HW descriptor:
1267 * - ststus bit TX_DMA_STATUS_DU is set by design,
1268 * so hardware will not try to process this desc.,
1269 * - rest of descriptor will be initialized on Tx.
1271 vring
->swtail
= wil_vring_next_tail(vring
);
1276 /* performance monitoring */
1277 used_new
= wil_vring_used_tx(vring
);
1278 if (wil_val_in_range(vring_idle_trsh
,
1279 used_new
, used_before_complete
)) {
1280 wil_dbg_txrx(wil
, "Ring[%2d] idle %d -> %d\n",
1281 ringid
, used_before_complete
, used_new
);
1282 txdata
->last_idle
= get_cycles();
1285 if (wil_vring_avail_tx(vring
) > wil_vring_wmark_high(vring
)) {
1286 wil_dbg_txrx(wil
, "netif_tx_wake : ring not full\n");
1287 netif_tx_wake_all_queues(wil_to_ndev(wil
));