2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/ipv6.h>
35 #include <linux/tcp.h>
36 #include <net/busy_poll.h>
39 static inline bool mlx5e_rx_hw_stamp(struct mlx5e_tstamp
*tstamp
)
41 return tstamp
->hwtstamp_config
.rx_filter
== HWTSTAMP_FILTER_ALL
;
44 static inline int mlx5e_alloc_rx_wqe(struct mlx5e_rq
*rq
,
45 struct mlx5e_rx_wqe
*wqe
, u16 ix
)
50 skb
= netdev_alloc_skb(rq
->netdev
, rq
->wqe_sz
);
54 dma_addr
= dma_map_single(rq
->pdev
,
55 /* hw start padding */
61 if (unlikely(dma_mapping_error(rq
->pdev
, dma_addr
)))
64 skb_reserve(skb
, MLX5E_NET_IP_ALIGN
);
66 *((dma_addr_t
*)skb
->cb
) = dma_addr
;
67 wqe
->data
.addr
= cpu_to_be64(dma_addr
+ MLX5E_NET_IP_ALIGN
);
79 bool mlx5e_post_rx_wqes(struct mlx5e_rq
*rq
)
81 struct mlx5_wq_ll
*wq
= &rq
->wq
;
83 if (unlikely(!test_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE
, &rq
->state
)))
86 while (!mlx5_wq_ll_is_full(wq
)) {
87 struct mlx5e_rx_wqe
*wqe
= mlx5_wq_ll_get_wqe(wq
, wq
->head
);
89 if (unlikely(mlx5e_alloc_rx_wqe(rq
, wqe
, wq
->head
)))
92 mlx5_wq_ll_push(wq
, be16_to_cpu(wqe
->next
.next_wqe_index
));
95 /* ensure wqes are visible to device before updating doorbell record */
98 mlx5_wq_ll_update_db_record(wq
);
100 return !mlx5_wq_ll_is_full(wq
);
103 static void mlx5e_lro_update_hdr(struct sk_buff
*skb
, struct mlx5_cqe64
*cqe
)
105 struct ethhdr
*eth
= (struct ethhdr
*)(skb
->data
);
106 struct iphdr
*ipv4
= (struct iphdr
*)(skb
->data
+ ETH_HLEN
);
107 struct ipv6hdr
*ipv6
= (struct ipv6hdr
*)(skb
->data
+ ETH_HLEN
);
110 u8 l4_hdr_type
= get_cqe_l4_hdr_type(cqe
);
111 int tcp_ack
= ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA
== l4_hdr_type
) ||
112 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA
== l4_hdr_type
));
114 u16 tot_len
= be32_to_cpu(cqe
->byte_cnt
) - ETH_HLEN
;
116 if (eth
->h_proto
== htons(ETH_P_IP
)) {
117 tcp
= (struct tcphdr
*)(skb
->data
+ ETH_HLEN
+
118 sizeof(struct iphdr
));
120 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
122 tcp
= (struct tcphdr
*)(skb
->data
+ ETH_HLEN
+
123 sizeof(struct ipv6hdr
));
125 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
128 if (get_cqe_lro_tcppsh(cqe
))
133 tcp
->ack_seq
= cqe
->lro_ack_seq_num
;
134 tcp
->window
= cqe
->lro_tcp_win
;
138 ipv4
->ttl
= cqe
->lro_min_ttl
;
139 ipv4
->tot_len
= cpu_to_be16(tot_len
);
141 ipv4
->check
= ip_fast_csum((unsigned char *)ipv4
,
144 ipv6
->hop_limit
= cqe
->lro_min_ttl
;
145 ipv6
->payload_len
= cpu_to_be16(tot_len
-
146 sizeof(struct ipv6hdr
));
150 static inline void mlx5e_skb_set_hash(struct mlx5_cqe64
*cqe
,
153 u8 cht
= cqe
->rss_hash_type
;
154 int ht
= (cht
& CQE_RSS_HTYPE_L4
) ? PKT_HASH_TYPE_L4
:
155 (cht
& CQE_RSS_HTYPE_IP
) ? PKT_HASH_TYPE_L3
:
157 skb_set_hash(skb
, be32_to_cpu(cqe
->rss_hash_result
), ht
);
160 static inline bool is_first_ethertype_ip(struct sk_buff
*skb
)
162 __be16 ethertype
= ((struct ethhdr
*)skb
->data
)->h_proto
;
164 return (ethertype
== htons(ETH_P_IP
) || ethertype
== htons(ETH_P_IPV6
));
167 static inline void mlx5e_handle_csum(struct net_device
*netdev
,
168 struct mlx5_cqe64
*cqe
,
172 if (unlikely(!(netdev
->features
& NETIF_F_RXCSUM
)))
175 if (likely(cqe
->hds_ip_ext
& CQE_L4_OK
)) {
176 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
177 } else if (is_first_ethertype_ip(skb
)) {
178 skb
->ip_summed
= CHECKSUM_COMPLETE
;
179 skb
->csum
= csum_unfold((__force __sum16
)cqe
->check_sum
);
188 skb
->ip_summed
= CHECKSUM_NONE
;
189 rq
->stats
.csum_none
++;
192 static inline void mlx5e_build_rx_skb(struct mlx5_cqe64
*cqe
,
196 struct net_device
*netdev
= rq
->netdev
;
197 u32 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
198 struct mlx5e_tstamp
*tstamp
= rq
->tstamp
;
201 skb_put(skb
, cqe_bcnt
);
203 lro_num_seg
= be32_to_cpu(cqe
->srqn
) >> 24;
204 if (lro_num_seg
> 1) {
205 mlx5e_lro_update_hdr(skb
, cqe
);
206 skb_shinfo(skb
)->gso_size
= DIV_ROUND_UP(cqe_bcnt
, lro_num_seg
);
207 rq
->stats
.lro_packets
++;
208 rq
->stats
.lro_bytes
+= cqe_bcnt
;
211 if (unlikely(mlx5e_rx_hw_stamp(tstamp
)))
212 mlx5e_fill_hwstamp(tstamp
, get_cqe_ts(cqe
), skb_hwtstamps(skb
));
214 mlx5e_handle_csum(netdev
, cqe
, rq
, skb
);
216 skb
->protocol
= eth_type_trans(skb
, netdev
);
218 skb_record_rx_queue(skb
, rq
->ix
);
220 if (likely(netdev
->features
& NETIF_F_RXHASH
))
221 mlx5e_skb_set_hash(cqe
, skb
);
223 if (cqe_has_vlan(cqe
))
224 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
225 be16_to_cpu(cqe
->vlan_info
));
228 int mlx5e_poll_rx_cq(struct mlx5e_cq
*cq
, int budget
)
230 struct mlx5e_rq
*rq
= container_of(cq
, struct mlx5e_rq
, cq
);
233 /* avoid accessing cq (dma coherent memory) if not needed */
234 if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES
, &cq
->flags
))
237 for (work_done
= 0; work_done
< budget
; work_done
++) {
238 struct mlx5e_rx_wqe
*wqe
;
239 struct mlx5_cqe64
*cqe
;
241 __be16 wqe_counter_be
;
244 cqe
= mlx5e_get_cqe(cq
);
248 mlx5_cqwq_pop(&cq
->wq
);
250 wqe_counter_be
= cqe
->wqe_counter
;
251 wqe_counter
= be16_to_cpu(wqe_counter_be
);
252 wqe
= mlx5_wq_ll_get_wqe(&rq
->wq
, wqe_counter
);
253 skb
= rq
->skb
[wqe_counter
];
255 rq
->skb
[wqe_counter
] = NULL
;
257 dma_unmap_single(rq
->pdev
,
258 *((dma_addr_t
*)skb
->cb
),
262 if (unlikely((cqe
->op_own
>> 4) != MLX5_CQE_RESP_SEND
)) {
268 mlx5e_build_rx_skb(cqe
, rq
, skb
);
270 napi_gro_receive(cq
->napi
, skb
);
273 mlx5_wq_ll_pop(&rq
->wq
, wqe_counter_be
,
274 &wqe
->next
.next_wqe_index
);
277 mlx5_cqwq_update_db_record(&cq
->wq
);
279 /* ensure cq space is freed before enabling more cqes */
282 if (work_done
== budget
)
283 set_bit(MLX5E_CQ_HAS_CQES
, &cq
->flags
);