projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'topic/omap3isp' of git://git.kernel.org/pub/scm/linux/kernel/git/mcheha...
[deliverable/linux.git] / drivers / net / ethernet / intel / i40e / i40e_txrx.c
1 /*******************************************************************************
2 *
3 * Intel Ethernet Controller XL710 Family Linux Driver
4 * Copyright(c) 2013 - 2014 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program. If not, see <http://www.gnu.org/licenses/>.
17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 * Contact Information:
22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
24 *
25 ******************************************************************************/
26
27 #include <linux/prefetch.h>
28 #include "i40e.h"
29 #include "i40e_prototype.h"
30
31 static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
32 u32 td_tag)
33 {
34 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
35 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
36 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
37 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
38 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
39 }
40
41 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
42 /**
43 * i40e_program_fdir_filter - Program a Flow Director filter
44 * @fdir_data: Packet data that will be filter parameters
45 * @raw_packet: the pre-allocated packet buffer for FDir
46 * @pf: The pf pointer
47 * @add: True for add/update, False for remove
48 **/
49 int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
50 struct i40e_pf *pf, bool add)
51 {
52 struct i40e_filter_program_desc *fdir_desc;
53 struct i40e_tx_buffer *tx_buf;
54 struct i40e_tx_desc *tx_desc;
55 struct i40e_ring *tx_ring;
56 unsigned int fpt, dcc;
57 struct i40e_vsi *vsi;
58 struct device *dev;
59 dma_addr_t dma;
60 u32 td_cmd = 0;
61 u16 i;
62
63 /* find existing FDIR VSI */
64 vsi = NULL;
65 for (i = 0; i < pf->num_alloc_vsi; i++)
66 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
67 vsi = pf->vsi[i];
68 if (!vsi)
69 return -ENOENT;
70
71 tx_ring = vsi->tx_rings[0];
72 dev = tx_ring->dev;
73
74 dma = dma_map_single(dev, raw_packet,
75 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
76 if (dma_mapping_error(dev, dma))
77 goto dma_fail;
78
79 /* grab the next descriptor */
80 i = tx_ring->next_to_use;
81 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
82
83 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
84
85 fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
86 I40E_TXD_FLTR_QW0_QINDEX_MASK;
87
88 fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
89 I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
90
91 fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
92 I40E_TXD_FLTR_QW0_PCTYPE_MASK;
93
94 /* Use LAN VSI Id if not programmed by user */
95 if (fdir_data->dest_vsi == 0)
96 fpt |= (pf->vsi[pf->lan_vsi]->id) <<
97 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
98 else
99 fpt |= ((u32)fdir_data->dest_vsi <<
100 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
101 I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
102
103 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
104
105 dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
106
107 if (add)
108 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
109 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
110 else
111 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
112 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
113
114 dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
115 I40E_TXD_FLTR_QW1_DEST_MASK;
116
117 dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
118 I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
119
120 if (fdir_data->cnt_index != 0) {
121 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
122 dcc |= ((u32)fdir_data->cnt_index <<
123 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
124 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
125 }
126
127 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
128 fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
129
130 /* Now program a dummy descriptor */
131 i = tx_ring->next_to_use;
132 tx_desc = I40E_TX_DESC(tx_ring, i);
133 tx_buf = &tx_ring->tx_bi[i];
134
135 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
136
137 /* record length, and DMA address */
138 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
139 dma_unmap_addr_set(tx_buf, dma, dma);
140
141 tx_desc->buffer_addr = cpu_to_le64(dma);
142 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
143
144 tx_desc->cmd_type_offset_bsz =
145 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
146
147 /* set the timestamp */
148 tx_buf->time_stamp = jiffies;
149
150 /* Force memory writes to complete before letting h/w
151 * know there are new descriptors to fetch. (Only
152 * applicable for weak-ordered memory model archs,
153 * such as IA-64).
154 */
155 wmb();
156
157 /* Mark the data descriptor to be watched */
158 tx_buf->next_to_watch = tx_desc;
159
160 writel(tx_ring->next_to_use, tx_ring->tail);
161 return 0;
162
163 dma_fail:
164 return -1;
165 }
166
167 #define IP_HEADER_OFFSET 14
168 #define I40E_UDPIP_DUMMY_PACKET_LEN 42
169 /**
170 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters
171 * @vsi: pointer to the targeted VSI
172 * @fd_data: the flow director data required for the FDir descriptor
173 * @raw_packet: the pre-allocated packet buffer for FDir
174 * @add: true adds a filter, false removes it
175 *
176 * Returns 0 if the filters were successfully added or removed
177 **/
178 static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi,
179 struct i40e_fdir_filter *fd_data,
180 u8 *raw_packet, bool add)
181 {
182 struct i40e_pf *pf = vsi->back;
183 struct udphdr *udp;
184 struct iphdr *ip;
185 bool err = false;
186 int ret;
187 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
188 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0,
189 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
190
191 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN);
192
193 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
194 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET
195 + sizeof(struct iphdr));
196
197 ip->daddr = fd_data->dst_ip[0];
198 udp->dest = fd_data->dst_port;
199 ip->saddr = fd_data->src_ip[0];
200 udp->source = fd_data->src_port;
201
202 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
203 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
204 if (ret) {
205 dev_info(&pf->pdev->dev,
206 "Filter command send failed for PCTYPE %d (ret = %d)\n",
207 fd_data->pctype, ret);
208 err = true;
209 } else {
210 dev_info(&pf->pdev->dev,
211 "Filter OK for PCTYPE %d (ret = %d)\n",
212 fd_data->pctype, ret);
213 }
214
215 return err ? -EOPNOTSUPP : 0;
216 }
217
218 #define I40E_TCPIP_DUMMY_PACKET_LEN 54
219 /**
220 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters
221 * @vsi: pointer to the targeted VSI
222 * @fd_data: the flow director data required for the FDir descriptor
223 * @raw_packet: the pre-allocated packet buffer for FDir
224 * @add: true adds a filter, false removes it
225 *
226 * Returns 0 if the filters were successfully added or removed
227 **/
228 static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
229 struct i40e_fdir_filter *fd_data,
230 u8 *raw_packet, bool add)
231 {
232 struct i40e_pf *pf = vsi->back;
233 struct tcphdr *tcp;
234 struct iphdr *ip;
235 bool err = false;
236 int ret;
237 /* Dummy packet */
238 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
239 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0,
240 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11,
241 0x0, 0x72, 0, 0, 0, 0};
242
243 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN);
244
245 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
246 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET
247 + sizeof(struct iphdr));
248
249 ip->daddr = fd_data->dst_ip[0];
250 tcp->dest = fd_data->dst_port;
251 ip->saddr = fd_data->src_ip[0];
252 tcp->source = fd_data->src_port;
253
254 if (add) {
255 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
256 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
257 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
258 }
259 }
260
261 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
262 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
263
264 if (ret) {
265 dev_info(&pf->pdev->dev,
266 "Filter command send failed for PCTYPE %d (ret = %d)\n",
267 fd_data->pctype, ret);
268 err = true;
269 } else {
270 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
271 fd_data->pctype, ret);
272 }
273
274 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
275
276 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
277 if (ret) {
278 dev_info(&pf->pdev->dev,
279 "Filter command send failed for PCTYPE %d (ret = %d)\n",
280 fd_data->pctype, ret);
281 err = true;
282 } else {
283 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
284 fd_data->pctype, ret);
285 }
286
287 return err ? -EOPNOTSUPP : 0;
288 }
289
290 /**
291 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for
292 * a specific flow spec
293 * @vsi: pointer to the targeted VSI
294 * @fd_data: the flow director data required for the FDir descriptor
295 * @raw_packet: the pre-allocated packet buffer for FDir
296 * @add: true adds a filter, false removes it
297 *
298 * Always returns -EOPNOTSUPP
299 **/
300 static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi,
301 struct i40e_fdir_filter *fd_data,
302 u8 *raw_packet, bool add)
303 {
304 return -EOPNOTSUPP;
305 }
306
307 #define I40E_IP_DUMMY_PACKET_LEN 34
308 /**
309 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for
310 * a specific flow spec
311 * @vsi: pointer to the targeted VSI
312 * @fd_data: the flow director data required for the FDir descriptor
313 * @raw_packet: the pre-allocated packet buffer for FDir
314 * @add: true adds a filter, false removes it
315 *
316 * Returns 0 if the filters were successfully added or removed
317 **/
318 static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
319 struct i40e_fdir_filter *fd_data,
320 u8 *raw_packet, bool add)
321 {
322 struct i40e_pf *pf = vsi->back;
323 struct iphdr *ip;
324 bool err = false;
325 int ret;
326 int i;
327 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
328 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0,
329 0, 0, 0, 0};
330
331 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN);
332 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
333
334 ip->saddr = fd_data->src_ip[0];
335 ip->daddr = fd_data->dst_ip[0];
336 ip->protocol = 0;
337
338 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
339 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) {
340 fd_data->pctype = i;
341 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
342
343 if (ret) {
344 dev_info(&pf->pdev->dev,
345 "Filter command send failed for PCTYPE %d (ret = %d)\n",
346 fd_data->pctype, ret);
347 err = true;
348 } else {
349 dev_info(&pf->pdev->dev,
350 "Filter OK for PCTYPE %d (ret = %d)\n",
351 fd_data->pctype, ret);
352 }
353 }
354
355 return err ? -EOPNOTSUPP : 0;
356 }
357
358 /**
359 * i40e_add_del_fdir - Build raw packets to add/del fdir filter
360 * @vsi: pointer to the targeted VSI
361 * @cmd: command to get or set RX flow classification rules
362 * @add: true adds a filter, false removes it
363 *
364 **/
365 int i40e_add_del_fdir(struct i40e_vsi *vsi,
366 struct i40e_fdir_filter *input, bool add)
367 {
368 struct i40e_pf *pf = vsi->back;
369 u8 *raw_packet;
370 int ret;
371
372 /* Populate the Flow Director that we have at the moment
373 * and allocate the raw packet buffer for the calling functions
374 */
375 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
376 if (!raw_packet)
377 return -ENOMEM;
378
379 switch (input->flow_type & ~FLOW_EXT) {
380 case TCP_V4_FLOW:
381 ret = i40e_add_del_fdir_tcpv4(vsi, input, raw_packet,
382 add);
383 break;
384 case UDP_V4_FLOW:
385 ret = i40e_add_del_fdir_udpv4(vsi, input, raw_packet,
386 add);
387 break;
388 case SCTP_V4_FLOW:
389 ret = i40e_add_del_fdir_sctpv4(vsi, input, raw_packet,
390 add);
391 break;
392 case IPV4_FLOW:
393 ret = i40e_add_del_fdir_ipv4(vsi, input, raw_packet,
394 add);
395 break;
396 case IP_USER_FLOW:
397 switch (input->ip4_proto) {
398 case IPPROTO_TCP:
399 ret = i40e_add_del_fdir_tcpv4(vsi, input,
400 raw_packet, add);
401 break;
402 case IPPROTO_UDP:
403 ret = i40e_add_del_fdir_udpv4(vsi, input,
404 raw_packet, add);
405 break;
406 case IPPROTO_SCTP:
407 ret = i40e_add_del_fdir_sctpv4(vsi, input,
408 raw_packet, add);
409 break;
410 default:
411 ret = i40e_add_del_fdir_ipv4(vsi, input,
412 raw_packet, add);
413 break;
414 }
415 break;
416 default:
417 dev_info(&pf->pdev->dev, "Could not specify spec type %d\n",
418 input->flow_type);
419 ret = -EINVAL;
420 }
421
422 kfree(raw_packet);
423 return ret;
424 }
425
426 /**
427 * i40e_fd_handle_status - check the Programming Status for FD
428 * @rx_ring: the Rx ring for this descriptor
429 * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor.
430 * @prog_id: the id originally used for programming
431 *
432 * This is used to verify if the FD programming or invalidation
433 * requested by SW to the HW is successful or not and take actions accordingly.
434 **/
435 static void i40e_fd_handle_status(struct i40e_ring *rx_ring,
436 union i40e_rx_desc *rx_desc, u8 prog_id)
437 {
438 struct i40e_pf *pf = rx_ring->vsi->back;
439 struct pci_dev *pdev = pf->pdev;
440 u32 fcnt_prog, fcnt_avail;
441 u32 error;
442 u64 qw;
443
444 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
445 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
446 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
447
448 if (error == (0x1 << I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
449 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
450 rx_desc->wb.qword0.hi_dword.fd_id);
451
452 /* filter programming failed most likely due to table full */
453 fcnt_prog = i40e_get_current_fd_count(pf);
454 fcnt_avail = i40e_get_fd_cnt_all(pf);
455 /* If ATR is running fcnt_prog can quickly change,
456 * if we are very close to full, it makes sense to disable
457 * FD ATR/SB and then re-enable it when there is room.
458 */
459 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) {
460 /* Turn off ATR first */
461 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
462 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
463 dev_warn(&pdev->dev, "FD filter space full, ATR for further flows will be turned off\n");
464 pf->auto_disable_flags |=
465 I40E_FLAG_FD_ATR_ENABLED;
466 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
467 } else if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
468 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
469 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n");
470 pf->auto_disable_flags |=
471 I40E_FLAG_FD_SB_ENABLED;
472 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
473 }
474 } else {
475 dev_info(&pdev->dev, "FD filter programming error\n");
476 }
477 } else if (error ==
478 (0x1 << I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
479 if (I40E_DEBUG_FD & pf->hw.debug_mask)
480 dev_info(&pdev->dev, "ntuple filter loc = %d, could not be removed\n",
481 rx_desc->wb.qword0.hi_dword.fd_id);
482 }
483 }
484
485 /**
486 * i40e_unmap_and_free_tx_resource - Release a Tx buffer
487 * @ring: the ring that owns the buffer
488 * @tx_buffer: the buffer to free
489 **/
490 static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
491 struct i40e_tx_buffer *tx_buffer)
492 {
493 if (tx_buffer->skb) {
494 dev_kfree_skb_any(tx_buffer->skb);
495 if (dma_unmap_len(tx_buffer, len))
496 dma_unmap_single(ring->dev,
497 dma_unmap_addr(tx_buffer, dma),
498 dma_unmap_len(tx_buffer, len),
499 DMA_TO_DEVICE);
500 } else if (dma_unmap_len(tx_buffer, len)) {
501 dma_unmap_page(ring->dev,
502 dma_unmap_addr(tx_buffer, dma),
503 dma_unmap_len(tx_buffer, len),
504 DMA_TO_DEVICE);
505 }
506 tx_buffer->next_to_watch = NULL;
507 tx_buffer->skb = NULL;
508 dma_unmap_len_set(tx_buffer, len, 0);
509 /* tx_buffer must be completely set up in the transmit path */
510 }
511
512 /**
513 * i40e_clean_tx_ring - Free any empty Tx buffers
514 * @tx_ring: ring to be cleaned
515 **/
516 void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
517 {
518 unsigned long bi_size;
519 u16 i;
520
521 /* ring already cleared, nothing to do */
522 if (!tx_ring->tx_bi)
523 return;
524
525 /* Free all the Tx ring sk_buffs */
526 for (i = 0; i < tx_ring->count; i++)
527 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
528
529 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
530 memset(tx_ring->tx_bi, 0, bi_size);
531
532 /* Zero out the descriptor ring */
533 memset(tx_ring->desc, 0, tx_ring->size);
534
535 tx_ring->next_to_use = 0;
536 tx_ring->next_to_clean = 0;
537
538 if (!tx_ring->netdev)
539 return;
540
541 /* cleanup Tx queue statistics */
542 netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
543 tx_ring->queue_index));
544 }
545
546 /**
547 * i40e_free_tx_resources - Free Tx resources per queue
548 * @tx_ring: Tx descriptor ring for a specific queue
549 *
550 * Free all transmit software resources
551 **/
552 void i40e_free_tx_resources(struct i40e_ring *tx_ring)
553 {
554 i40e_clean_tx_ring(tx_ring);
555 kfree(tx_ring->tx_bi);
556 tx_ring->tx_bi = NULL;
557
558 if (tx_ring->desc) {
559 dma_free_coherent(tx_ring->dev, tx_ring->size,
560 tx_ring->desc, tx_ring->dma);
561 tx_ring->desc = NULL;
562 }
563 }
564
565 /**
566 * i40e_get_tx_pending - how many tx descriptors not processed
567 * @tx_ring: the ring of descriptors
568 *
569 * Since there is no access to the ring head register
570 * in XL710, we need to use our local copies
571 **/
572 static u32 i40e_get_tx_pending(struct i40e_ring *ring)
573 {
574 u32 ntu = ((ring->next_to_clean <= ring->next_to_use)
575 ? ring->next_to_use
576 : ring->next_to_use + ring->count);
577 return ntu - ring->next_to_clean;
578 }
579
580 /**
581 * i40e_check_tx_hang - Is there a hang in the Tx queue
582 * @tx_ring: the ring of descriptors
583 **/
584 static bool i40e_check_tx_hang(struct i40e_ring *tx_ring)
585 {
586 u32 tx_pending = i40e_get_tx_pending(tx_ring);
587 bool ret = false;
588
589 clear_check_for_tx_hang(tx_ring);
590
591 /* Check for a hung queue, but be thorough. This verifies
592 * that a transmit has been completed since the previous
593 * check AND there is at least one packet pending. The
594 * ARMED bit is set to indicate a potential hang. The
595 * bit is cleared if a pause frame is received to remove
596 * false hang detection due to PFC or 802.3x frames. By
597 * requiring this to fail twice we avoid races with
598 * PFC clearing the ARMED bit and conditions where we
599 * run the check_tx_hang logic with a transmit completion
600 * pending but without time to complete it yet.
601 */
602 if ((tx_ring->tx_stats.tx_done_old == tx_ring->stats.packets) &&
603 tx_pending) {
604 /* make sure it is true for two checks in a row */
605 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED,
606 &tx_ring->state);
607 } else {
608 /* update completed stats and disarm the hang check */
609 tx_ring->tx_stats.tx_done_old = tx_ring->stats.packets;
610 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state);
611 }
612
613 return ret;
614 }
615
616 /**
617 * i40e_get_head - Retrieve head from head writeback
618 * @tx_ring: tx ring to fetch head of
619 *
620 * Returns value of Tx ring head based on value stored
621 * in head write-back location
622 **/
623 static inline u32 i40e_get_head(struct i40e_ring *tx_ring)
624 {
625 void *head = (struct i40e_tx_desc *)tx_ring->desc + tx_ring->count;
626
627 return le32_to_cpu(*(volatile __le32 *)head);
628 }
629
630 /**
631 * i40e_clean_tx_irq - Reclaim resources after transmit completes
632 * @tx_ring: tx ring to clean
633 * @budget: how many cleans we're allowed
634 *
635 * Returns true if there's any budget left (e.g. the clean is finished)
636 **/
637 static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
638 {
639 u16 i = tx_ring->next_to_clean;
640 struct i40e_tx_buffer *tx_buf;
641 struct i40e_tx_desc *tx_head;
642 struct i40e_tx_desc *tx_desc;
643 unsigned int total_packets = 0;
644 unsigned int total_bytes = 0;
645
646 tx_buf = &tx_ring->tx_bi[i];
647 tx_desc = I40E_TX_DESC(tx_ring, i);
648 i -= tx_ring->count;
649
650 tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring));
651
652 do {
653 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
654
655 /* if next_to_watch is not set then there is no work pending */
656 if (!eop_desc)
657 break;
658
659 /* prevent any other reads prior to eop_desc */
660 read_barrier_depends();
661
662 /* we have caught up to head, no work left to do */
663 if (tx_head == tx_desc)
664 break;
665
666 /* clear next_to_watch to prevent false hangs */
667 tx_buf->next_to_watch = NULL;
668
669 /* update the statistics for this packet */
670 total_bytes += tx_buf->bytecount;
671 total_packets += tx_buf->gso_segs;
672
673 /* free the skb */
674 dev_kfree_skb_any(tx_buf->skb);
675
676 /* unmap skb header data */
677 dma_unmap_single(tx_ring->dev,
678 dma_unmap_addr(tx_buf, dma),
679 dma_unmap_len(tx_buf, len),
680 DMA_TO_DEVICE);
681
682 /* clear tx_buffer data */
683 tx_buf->skb = NULL;
684 dma_unmap_len_set(tx_buf, len, 0);
685
686 /* unmap remaining buffers */
687 while (tx_desc != eop_desc) {
688
689 tx_buf++;
690 tx_desc++;
691 i++;
692 if (unlikely(!i)) {
693 i -= tx_ring->count;
694 tx_buf = tx_ring->tx_bi;
695 tx_desc = I40E_TX_DESC(tx_ring, 0);
696 }
697
698 /* unmap any remaining paged data */
699 if (dma_unmap_len(tx_buf, len)) {
700 dma_unmap_page(tx_ring->dev,
701 dma_unmap_addr(tx_buf, dma),
702 dma_unmap_len(tx_buf, len),
703 DMA_TO_DEVICE);
704 dma_unmap_len_set(tx_buf, len, 0);
705 }
706 }
707
708 /* move us one more past the eop_desc for start of next pkt */
709 tx_buf++;
710 tx_desc++;
711 i++;
712 if (unlikely(!i)) {
713 i -= tx_ring->count;
714 tx_buf = tx_ring->tx_bi;
715 tx_desc = I40E_TX_DESC(tx_ring, 0);
716 }
717
718 /* update budget accounting */
719 budget--;
720 } while (likely(budget));
721
722 i += tx_ring->count;
723 tx_ring->next_to_clean = i;
724 u64_stats_update_begin(&tx_ring->syncp);
725 tx_ring->stats.bytes += total_bytes;
726 tx_ring->stats.packets += total_packets;
727 u64_stats_update_end(&tx_ring->syncp);
728 tx_ring->q_vector->tx.total_bytes += total_bytes;
729 tx_ring->q_vector->tx.total_packets += total_packets;
730
731 if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) {
732 /* schedule immediate reset if we believe we hung */
733 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n"
734 " VSI <%d>\n"
735 " Tx Queue <%d>\n"
736 " next_to_use <%x>\n"
737 " next_to_clean <%x>\n",
738 tx_ring->vsi->seid,
739 tx_ring->queue_index,
740 tx_ring->next_to_use, i);
741 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n"
742 " time_stamp <%lx>\n"
743 " jiffies <%lx>\n",
744 tx_ring->tx_bi[i].time_stamp, jiffies);
745
746 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
747
748 dev_info(tx_ring->dev,
749 "tx hang detected on queue %d, resetting adapter\n",
750 tx_ring->queue_index);
751
752 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev);
753
754 /* the adapter is about to reset, no point in enabling stuff */
755 return true;
756 }
757
758 netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
759 tx_ring->queue_index),
760 total_packets, total_bytes);
761
762 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
763 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
764 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
765 /* Make sure that anybody stopping the queue after this
766 * sees the new next_to_clean.
767 */
768 smp_mb();
769 if (__netif_subqueue_stopped(tx_ring->netdev,
770 tx_ring->queue_index) &&
771 !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
772 netif_wake_subqueue(tx_ring->netdev,
773 tx_ring->queue_index);
774 ++tx_ring->tx_stats.restart_queue;
775 }
776 }
777
778 return budget > 0;
779 }
780
781 /**
782 * i40e_set_new_dynamic_itr - Find new ITR level
783 * @rc: structure containing ring performance data
784 *
785 * Stores a new ITR value based on packets and byte counts during
786 * the last interrupt. The advantage of per interrupt computation
787 * is faster updates and more accurate ITR for the current traffic
788 * pattern. Constants in this function were computed based on
789 * theoretical maximum wire speed and thresholds were set based on
790 * testing data as well as attempting to minimize response time
791 * while increasing bulk throughput.
792 **/
793 static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
794 {
795 enum i40e_latency_range new_latency_range = rc->latency_range;
796 u32 new_itr = rc->itr;
797 int bytes_per_int;
798
799 if (rc->total_packets == 0 || !rc->itr)
800 return;
801
802 /* simple throttlerate management
803 * 0-10MB/s lowest (100000 ints/s)
804 * 10-20MB/s low (20000 ints/s)
805 * 20-1249MB/s bulk (8000 ints/s)
806 */
807 bytes_per_int = rc->total_bytes / rc->itr;
808 switch (rc->itr) {
809 case I40E_LOWEST_LATENCY:
810 if (bytes_per_int > 10)
811 new_latency_range = I40E_LOW_LATENCY;
812 break;
813 case I40E_LOW_LATENCY:
814 if (bytes_per_int > 20)
815 new_latency_range = I40E_BULK_LATENCY;
816 else if (bytes_per_int <= 10)
817 new_latency_range = I40E_LOWEST_LATENCY;
818 break;
819 case I40E_BULK_LATENCY:
820 if (bytes_per_int <= 20)
821 rc->latency_range = I40E_LOW_LATENCY;
822 break;
823 }
824
825 switch (new_latency_range) {
826 case I40E_LOWEST_LATENCY:
827 new_itr = I40E_ITR_100K;
828 break;
829 case I40E_LOW_LATENCY:
830 new_itr = I40E_ITR_20K;
831 break;
832 case I40E_BULK_LATENCY:
833 new_itr = I40E_ITR_8K;
834 break;
835 default:
836 break;
837 }
838
839 if (new_itr != rc->itr) {
840 /* do an exponential smoothing */
841 new_itr = (10 * new_itr * rc->itr) /
842 ((9 * new_itr) + rc->itr);
843 rc->itr = new_itr & I40E_MAX_ITR;
844 }
845
846 rc->total_bytes = 0;
847 rc->total_packets = 0;
848 }
849
850 /**
851 * i40e_update_dynamic_itr - Adjust ITR based on bytes per int
852 * @q_vector: the vector to adjust
853 **/
854 static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector)
855 {
856 u16 vector = q_vector->vsi->base_vector + q_vector->v_idx;
857 struct i40e_hw *hw = &q_vector->vsi->back->hw;
858 u32 reg_addr;
859 u16 old_itr;
860
861 reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1);
862 old_itr = q_vector->rx.itr;
863 i40e_set_new_dynamic_itr(&q_vector->rx);
864 if (old_itr != q_vector->rx.itr)
865 wr32(hw, reg_addr, q_vector->rx.itr);
866
867 reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1);
868 old_itr = q_vector->tx.itr;
869 i40e_set_new_dynamic_itr(&q_vector->tx);
870 if (old_itr != q_vector->tx.itr)
871 wr32(hw, reg_addr, q_vector->tx.itr);
872 }
873
874 /**
875 * i40e_clean_programming_status - clean the programming status descriptor
876 * @rx_ring: the rx ring that has this descriptor
877 * @rx_desc: the rx descriptor written back by HW
878 *
879 * Flow director should handle FD_FILTER_STATUS to check its filter programming
880 * status being successful or not and take actions accordingly. FCoE should
881 * handle its context/filter programming/invalidation status and take actions.
882 *
883 **/
884 static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
885 union i40e_rx_desc *rx_desc)
886 {
887 u64 qw;
888 u8 id;
889
890 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
891 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
892 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
893
894 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
895 i40e_fd_handle_status(rx_ring, rx_desc, id);
896 }
897
898 /**
899 * i40e_setup_tx_descriptors - Allocate the Tx descriptors
900 * @tx_ring: the tx ring to set up
901 *
902 * Return 0 on success, negative on error
903 **/
904 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
905 {
906 struct device *dev = tx_ring->dev;
907 int bi_size;
908
909 if (!dev)
910 return -ENOMEM;
911
912 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
913 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
914 if (!tx_ring->tx_bi)
915 goto err;
916
917 /* round up to nearest 4K */
918 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
919 /* add u32 for head writeback, align after this takes care of
920 * guaranteeing this is at least one cache line in size
921 */
922 tx_ring->size += sizeof(u32);
923 tx_ring->size = ALIGN(tx_ring->size, 4096);
924 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
925 &tx_ring->dma, GFP_KERNEL);
926 if (!tx_ring->desc) {
927 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
928 tx_ring->size);
929 goto err;
930 }
931
932 tx_ring->next_to_use = 0;
933 tx_ring->next_to_clean = 0;
934 return 0;
935
936 err:
937 kfree(tx_ring->tx_bi);
938 tx_ring->tx_bi = NULL;
939 return -ENOMEM;
940 }
941
942 /**
943 * i40e_clean_rx_ring - Free Rx buffers
944 * @rx_ring: ring to be cleaned
945 **/
946 void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
947 {
948 struct device *dev = rx_ring->dev;
949 struct i40e_rx_buffer *rx_bi;
950 unsigned long bi_size;
951 u16 i;
952
953 /* ring already cleared, nothing to do */
954 if (!rx_ring->rx_bi)
955 return;
956
957 /* Free all the Rx ring sk_buffs */
958 for (i = 0; i < rx_ring->count; i++) {
959 rx_bi = &rx_ring->rx_bi[i];
960 if (rx_bi->dma) {
961 dma_unmap_single(dev,
962 rx_bi->dma,
963 rx_ring->rx_buf_len,
964 DMA_FROM_DEVICE);
965 rx_bi->dma = 0;
966 }
967 if (rx_bi->skb) {
968 dev_kfree_skb(rx_bi->skb);
969 rx_bi->skb = NULL;
970 }
971 if (rx_bi->page) {
972 if (rx_bi->page_dma) {
973 dma_unmap_page(dev,
974 rx_bi->page_dma,
975 PAGE_SIZE / 2,
976 DMA_FROM_DEVICE);
977 rx_bi->page_dma = 0;
978 }
979 __free_page(rx_bi->page);
980 rx_bi->page = NULL;
981 rx_bi->page_offset = 0;
982 }
983 }
984
985 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
986 memset(rx_ring->rx_bi, 0, bi_size);
987
988 /* Zero out the descriptor ring */
989 memset(rx_ring->desc, 0, rx_ring->size);
990
991 rx_ring->next_to_clean = 0;
992 rx_ring->next_to_use = 0;
993 }
994
995 /**
996 * i40e_free_rx_resources - Free Rx resources
997 * @rx_ring: ring to clean the resources from
998 *
999 * Free all receive software resources
1000 **/
1001 void i40e_free_rx_resources(struct i40e_ring *rx_ring)
1002 {
1003 i40e_clean_rx_ring(rx_ring);
1004 kfree(rx_ring->rx_bi);
1005 rx_ring->rx_bi = NULL;
1006
1007 if (rx_ring->desc) {
1008 dma_free_coherent(rx_ring->dev, rx_ring->size,
1009 rx_ring->desc, rx_ring->dma);
1010 rx_ring->desc = NULL;
1011 }
1012 }
1013
1014 /**
1015 * i40e_setup_rx_descriptors - Allocate Rx descriptors
1016 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
1017 *
1018 * Returns 0 on success, negative on failure
1019 **/
1020 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
1021 {
1022 struct device *dev = rx_ring->dev;
1023 int bi_size;
1024
1025 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
1026 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
1027 if (!rx_ring->rx_bi)
1028 goto err;
1029
1030 /* Round up to nearest 4K */
1031 rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
1032 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
1033 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
1034 rx_ring->size = ALIGN(rx_ring->size, 4096);
1035 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
1036 &rx_ring->dma, GFP_KERNEL);
1037
1038 if (!rx_ring->desc) {
1039 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
1040 rx_ring->size);
1041 goto err;
1042 }
1043
1044 rx_ring->next_to_clean = 0;
1045 rx_ring->next_to_use = 0;
1046
1047 return 0;
1048 err:
1049 kfree(rx_ring->rx_bi);
1050 rx_ring->rx_bi = NULL;
1051 return -ENOMEM;
1052 }
1053
1054 /**
1055 * i40e_release_rx_desc - Store the new tail and head values
1056 * @rx_ring: ring to bump
1057 * @val: new head index
1058 **/
1059 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1060 {
1061 rx_ring->next_to_use = val;
1062 /* Force memory writes to complete before letting h/w
1063 * know there are new descriptors to fetch. (Only
1064 * applicable for weak-ordered memory model archs,
1065 * such as IA-64).
1066 */
1067 wmb();
1068 writel(val, rx_ring->tail);
1069 }
1070
1071 /**
1072 * i40e_alloc_rx_buffers - Replace used receive buffers; packet split
1073 * @rx_ring: ring to place buffers on
1074 * @cleaned_count: number of buffers to replace
1075 **/
1076 void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
1077 {
1078 u16 i = rx_ring->next_to_use;
1079 union i40e_rx_desc *rx_desc;
1080 struct i40e_rx_buffer *bi;
1081 struct sk_buff *skb;
1082
1083 /* do nothing if no valid netdev defined */
1084 if (!rx_ring->netdev || !cleaned_count)
1085 return;
1086
1087 while (cleaned_count--) {
1088 rx_desc = I40E_RX_DESC(rx_ring, i);
1089 bi = &rx_ring->rx_bi[i];
1090 skb = bi->skb;
1091
1092 if (!skb) {
1093 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1094 rx_ring->rx_buf_len);
1095 if (!skb) {
1096 rx_ring->rx_stats.alloc_buff_failed++;
1097 goto no_buffers;
1098 }
1099 /* initialize queue mapping */
1100 skb_record_rx_queue(skb, rx_ring->queue_index);
1101 bi->skb = skb;
1102 }
1103
1104 if (!bi->dma) {
1105 bi->dma = dma_map_single(rx_ring->dev,
1106 skb->data,
1107 rx_ring->rx_buf_len,
1108 DMA_FROM_DEVICE);
1109 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1110 rx_ring->rx_stats.alloc_buff_failed++;
1111 bi->dma = 0;
1112 goto no_buffers;
1113 }
1114 }
1115
1116 if (ring_is_ps_enabled(rx_ring)) {
1117 if (!bi->page) {
1118 bi->page = alloc_page(GFP_ATOMIC);
1119 if (!bi->page) {
1120 rx_ring->rx_stats.alloc_page_failed++;
1121 goto no_buffers;
1122 }
1123 }
1124
1125 if (!bi->page_dma) {
1126 /* use a half page if we're re-using */
1127 bi->page_offset ^= PAGE_SIZE / 2;
1128 bi->page_dma = dma_map_page(rx_ring->dev,
1129 bi->page,
1130 bi->page_offset,
1131 PAGE_SIZE / 2,
1132 DMA_FROM_DEVICE);
1133 if (dma_mapping_error(rx_ring->dev,
1134 bi->page_dma)) {
1135 rx_ring->rx_stats.alloc_page_failed++;
1136 bi->page_dma = 0;
1137 goto no_buffers;
1138 }
1139 }
1140
1141 /* Refresh the desc even if buffer_addrs didn't change
1142 * because each write-back erases this info.
1143 */
1144 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1145 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1146 } else {
1147 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1148 rx_desc->read.hdr_addr = 0;
1149 }
1150 i++;
1151 if (i == rx_ring->count)
1152 i = 0;
1153 }
1154
1155 no_buffers:
1156 if (rx_ring->next_to_use != i)
1157 i40e_release_rx_desc(rx_ring, i);
1158 }
1159
1160 /**
1161 * i40e_receive_skb - Send a completed packet up the stack
1162 * @rx_ring: rx ring in play
1163 * @skb: packet to send up
1164 * @vlan_tag: vlan tag for packet
1165 **/
1166 static void i40e_receive_skb(struct i40e_ring *rx_ring,
1167 struct sk_buff *skb, u16 vlan_tag)
1168 {
1169 struct i40e_q_vector *q_vector = rx_ring->q_vector;
1170 struct i40e_vsi *vsi = rx_ring->vsi;
1171 u64 flags = vsi->back->flags;
1172
1173 if (vlan_tag & VLAN_VID_MASK)
1174 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
1175
1176 if (flags & I40E_FLAG_IN_NETPOLL)
1177 netif_rx(skb);
1178 else
1179 napi_gro_receive(&q_vector->napi, skb);
1180 }
1181
1182 /**
1183 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1184 * @vsi: the VSI we care about
1185 * @skb: skb currently being received and modified
1186 * @rx_status: status value of last descriptor in packet
1187 * @rx_error: error value of last descriptor in packet
1188 * @rx_ptype: ptype value of last descriptor in packet
1189 **/
1190 static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1191 struct sk_buff *skb,
1192 u32 rx_status,
1193 u32 rx_error,
1194 u16 rx_ptype)
1195 {
1196 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(rx_ptype);
1197 bool ipv4 = false, ipv6 = false;
1198 bool ipv4_tunnel, ipv6_tunnel;
1199 __wsum rx_udp_csum;
1200 struct iphdr *iph;
1201 __sum16 csum;
1202
1203 ipv4_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT4_MAC_PAY3) &&
1204 (rx_ptype < I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4);
1205 ipv6_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT6_MAC_PAY3) &&
1206 (rx_ptype < I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4);
1207
1208 skb->encapsulation = ipv4_tunnel || ipv6_tunnel;
1209 skb->ip_summed = CHECKSUM_NONE;
1210
1211 /* Rx csum enabled and ip headers found? */
1212 if (!(vsi->netdev->features & NETIF_F_RXCSUM))
1213 return;
1214
1215 /* did the hardware decode the packet and checksum? */
1216 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
1217 return;
1218
1219 /* both known and outer_ip must be set for the below code to work */
1220 if (!(decoded.known && decoded.outer_ip))
1221 return;
1222
1223 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1224 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4)
1225 ipv4 = true;
1226 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1227 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6)
1228 ipv6 = true;
1229
1230 if (ipv4 &&
1231 (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
1232 (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))))
1233 goto checksum_fail;
1234
1235 /* likely incorrect csum if alternate IP extension headers found */
1236 if (ipv6 &&
1237 decoded.inner_prot == I40E_RX_PTYPE_INNER_PROT_TCP &&
1238 rx_error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT) &&
1239 rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1240 /* don't increment checksum err here, non-fatal err */
1241 return;
1242
1243 /* there was some L4 error, count error and punt packet to the stack */
1244 if (rx_error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))
1245 goto checksum_fail;
1246
1247 /* handle packets that were not able to be checksummed due
1248 * to arrival speed, in this case the stack can compute
1249 * the csum.
1250 */
1251 if (rx_error & (1 << I40E_RX_DESC_ERROR_PPRS_SHIFT))
1252 return;
1253
1254 /* If VXLAN traffic has an outer UDPv4 checksum we need to check
1255 * it in the driver, hardware does not do it for us.
1256 * Since L3L4P bit was set we assume a valid IHL value (>=5)
1257 * so the total length of IPv4 header is IHL*4 bytes
1258 * The UDP_0 bit *may* bet set if the *inner* header is UDP
1259 */
1260 if (ipv4_tunnel &&
1261 (decoded.inner_prot != I40E_RX_PTYPE_INNER_PROT_UDP) &&
1262 !(rx_status & (1 << I40E_RX_DESC_STATUS_UDP_0_SHIFT))) {
1263 skb->transport_header = skb->mac_header +
1264 sizeof(struct ethhdr) +
1265 (ip_hdr(skb)->ihl * 4);
1266
1267 /* Add 4 bytes for VLAN tagged packets */
1268 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) ||
1269 skb->protocol == htons(ETH_P_8021AD))
1270 ? VLAN_HLEN : 0;
1271
1272 rx_udp_csum = udp_csum(skb);
1273 iph = ip_hdr(skb);
1274 csum = csum_tcpudp_magic(
1275 iph->saddr, iph->daddr,
1276 (skb->len - skb_transport_offset(skb)),
1277 IPPROTO_UDP, rx_udp_csum);
1278
1279 if (udp_hdr(skb)->check != csum)
1280 goto checksum_fail;
1281 }
1282
1283 skb->ip_summed = CHECKSUM_UNNECESSARY;
1284
1285 return;
1286
1287 checksum_fail:
1288 vsi->back->hw_csum_rx_error++;
1289 }
1290
1291 /**
1292 * i40e_rx_hash - returns the hash value from the Rx descriptor
1293 * @ring: descriptor ring
1294 * @rx_desc: specific descriptor
1295 **/
1296 static inline u32 i40e_rx_hash(struct i40e_ring *ring,
1297 union i40e_rx_desc *rx_desc)
1298 {
1299 const __le64 rss_mask =
1300 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1301 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1302
1303 if ((ring->netdev->features & NETIF_F_RXHASH) &&
1304 (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask)
1305 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1306 else
1307 return 0;
1308 }
1309
1310 /**
1311 * i40e_ptype_to_hash - get a hash type
1312 * @ptype: the ptype value from the descriptor
1313 *
1314 * Returns a hash type to be used by skb_set_hash
1315 **/
1316 static inline enum pkt_hash_types i40e_ptype_to_hash(u8 ptype)
1317 {
1318 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1319
1320 if (!decoded.known)
1321 return PKT_HASH_TYPE_NONE;
1322
1323 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1324 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1325 return PKT_HASH_TYPE_L4;
1326 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1327 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1328 return PKT_HASH_TYPE_L3;
1329 else
1330 return PKT_HASH_TYPE_L2;
1331 }
1332
1333 /**
1334 * i40e_clean_rx_irq - Reclaim resources after receive completes
1335 * @rx_ring: rx ring to clean
1336 * @budget: how many cleans we're allowed
1337 *
1338 * Returns true if there's any budget left (e.g. the clean is finished)
1339 **/
1340 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1341 {
1342 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1343 u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
1344 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1345 const int current_node = numa_node_id();
1346 struct i40e_vsi *vsi = rx_ring->vsi;
1347 u16 i = rx_ring->next_to_clean;
1348 union i40e_rx_desc *rx_desc;
1349 u32 rx_error, rx_status;
1350 u8 rx_ptype;
1351 u64 qword;
1352
1353 if (budget <= 0)
1354 return 0;
1355
1356 rx_desc = I40E_RX_DESC(rx_ring, i);
1357 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1358 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1359 I40E_RXD_QW1_STATUS_SHIFT;
1360
1361 while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1362 union i40e_rx_desc *next_rxd;
1363 struct i40e_rx_buffer *rx_bi;
1364 struct sk_buff *skb;
1365 u16 vlan_tag;
1366 if (i40e_rx_is_programming_status(qword)) {
1367 i40e_clean_programming_status(rx_ring, rx_desc);
1368 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1369 goto next_desc;
1370 }
1371 rx_bi = &rx_ring->rx_bi[i];
1372 skb = rx_bi->skb;
1373 prefetch(skb->data);
1374
1375 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1376 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1377 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
1378 I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1379 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
1380 I40E_RXD_QW1_LENGTH_SPH_SHIFT;
1381
1382 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1383 I40E_RXD_QW1_ERROR_SHIFT;
1384 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1385 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1386
1387 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1388 I40E_RXD_QW1_PTYPE_SHIFT;
1389 rx_bi->skb = NULL;
1390
1391 /* This memory barrier is needed to keep us from reading
1392 * any other fields out of the rx_desc until we know the
1393 * STATUS_DD bit is set
1394 */
1395 rmb();
1396
1397 /* Get the header and possibly the whole packet
1398 * If this is an skb from previous receive dma will be 0
1399 */
1400 if (rx_bi->dma) {
1401 u16 len;
1402
1403 if (rx_hbo)
1404 len = I40E_RX_HDR_SIZE;
1405 else if (rx_sph)
1406 len = rx_header_len;
1407 else if (rx_packet_len)
1408 len = rx_packet_len; /* 1buf/no split found */
1409 else
1410 len = rx_header_len; /* split always mode */
1411
1412 skb_put(skb, len);
1413 dma_unmap_single(rx_ring->dev,
1414 rx_bi->dma,
1415 rx_ring->rx_buf_len,
1416 DMA_FROM_DEVICE);
1417 rx_bi->dma = 0;
1418 }
1419
1420 /* Get the rest of the data if this was a header split */
1421 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) {
1422
1423 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1424 rx_bi->page,
1425 rx_bi->page_offset,
1426 rx_packet_len);
1427
1428 skb->len += rx_packet_len;
1429 skb->data_len += rx_packet_len;
1430 skb->truesize += rx_packet_len;
1431
1432 if ((page_count(rx_bi->page) == 1) &&
1433 (page_to_nid(rx_bi->page) == current_node))
1434 get_page(rx_bi->page);
1435 else
1436 rx_bi->page = NULL;
1437
1438 dma_unmap_page(rx_ring->dev,
1439 rx_bi->page_dma,
1440 PAGE_SIZE / 2,
1441 DMA_FROM_DEVICE);
1442 rx_bi->page_dma = 0;
1443 }
1444 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1445
1446 if (unlikely(
1447 !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
1448 struct i40e_rx_buffer *next_buffer;
1449
1450 next_buffer = &rx_ring->rx_bi[i];
1451
1452 if (ring_is_ps_enabled(rx_ring)) {
1453 rx_bi->skb = next_buffer->skb;
1454 rx_bi->dma = next_buffer->dma;
1455 next_buffer->skb = skb;
1456 next_buffer->dma = 0;
1457 }
1458 rx_ring->rx_stats.non_eop_descs++;
1459 goto next_desc;
1460 }
1461
1462 /* ERR_MASK will only have valid bits if EOP set */
1463 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1464 dev_kfree_skb_any(skb);
1465 /* TODO: shouldn't we increment a counter indicating the
1466 * drop?
1467 */
1468 goto next_desc;
1469 }
1470
1471 skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc),
1472 i40e_ptype_to_hash(rx_ptype));
1473 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1474 i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1475 I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1476 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1477 rx_ring->last_rx_timestamp = jiffies;
1478 }
1479
1480 /* probably a little skewed due to removing CRC */
1481 total_rx_bytes += skb->len;
1482 total_rx_packets++;
1483
1484 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1485
1486 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1487
1488 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
1489 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1490 : 0;
1491 i40e_receive_skb(rx_ring, skb, vlan_tag);
1492
1493 rx_ring->netdev->last_rx = jiffies;
1494 budget--;
1495 next_desc:
1496 rx_desc->wb.qword1.status_error_len = 0;
1497 if (!budget)
1498 break;
1499
1500 cleaned_count++;
1501 /* return some buffers to hardware, one at a time is too slow */
1502 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1503 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1504 cleaned_count = 0;
1505 }
1506
1507 /* use prefetched values */
1508 rx_desc = next_rxd;
1509 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1510 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1511 I40E_RXD_QW1_STATUS_SHIFT;
1512 }
1513
1514 rx_ring->next_to_clean = i;
1515 u64_stats_update_begin(&rx_ring->syncp);
1516 rx_ring->stats.packets += total_rx_packets;
1517 rx_ring->stats.bytes += total_rx_bytes;
1518 u64_stats_update_end(&rx_ring->syncp);
1519 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1520 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1521
1522 if (cleaned_count)
1523 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1524
1525 return budget > 0;
1526 }
1527
1528 /**
1529 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1530 * @napi: napi struct with our devices info in it
1531 * @budget: amount of work driver is allowed to do this pass, in packets
1532 *
1533 * This function will clean all queues associated with a q_vector.
1534 *
1535 * Returns the amount of work done
1536 **/
1537 int i40e_napi_poll(struct napi_struct *napi, int budget)
1538 {
1539 struct i40e_q_vector *q_vector =
1540 container_of(napi, struct i40e_q_vector, napi);
1541 struct i40e_vsi *vsi = q_vector->vsi;
1542 struct i40e_ring *ring;
1543 bool clean_complete = true;
1544 int budget_per_ring;
1545
1546 if (test_bit(__I40E_DOWN, &vsi->state)) {
1547 napi_complete(napi);
1548 return 0;
1549 }
1550
1551 /* Since the actual Tx work is minimal, we can give the Tx a larger
1552 * budget and be more aggressive about cleaning up the Tx descriptors.
1553 */
1554 i40e_for_each_ring(ring, q_vector->tx)
1555 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit);
1556
1557 /* We attempt to distribute budget to each Rx queue fairly, but don't
1558 * allow the budget to go below 1 because that would exit polling early.
1559 */
1560 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
1561
1562 i40e_for_each_ring(ring, q_vector->rx)
1563 clean_complete &= i40e_clean_rx_irq(ring, budget_per_ring);
1564
1565 /* If work not completed, return budget and polling will return */
1566 if (!clean_complete)
1567 return budget;
1568
1569 /* Work is done so exit the polling mode and re-enable the interrupt */
1570 napi_complete(napi);
1571 if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) ||
1572 ITR_IS_DYNAMIC(vsi->tx_itr_setting))
1573 i40e_update_dynamic_itr(q_vector);
1574
1575 if (!test_bit(__I40E_DOWN, &vsi->state)) {
1576 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1577 i40e_irq_dynamic_enable(vsi,
1578 q_vector->v_idx + vsi->base_vector);
1579 } else {
1580 struct i40e_hw *hw = &vsi->back->hw;
1581 /* We re-enable the queue 0 cause, but
1582 * don't worry about dynamic_enable
1583 * because we left it on for the other
1584 * possible interrupts during napi
1585 */
1586 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
1587 qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1588 wr32(hw, I40E_QINT_RQCTL(0), qval);
1589
1590 qval = rd32(hw, I40E_QINT_TQCTL(0));
1591 qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1592 wr32(hw, I40E_QINT_TQCTL(0), qval);
1593
1594 i40e_irq_dynamic_enable_icr0(vsi->back);
1595 }
1596 }
1597
1598 return 0;
1599 }
1600
1601 /**
1602 * i40e_atr - Add a Flow Director ATR filter
1603 * @tx_ring: ring to add programming descriptor to
1604 * @skb: send buffer
1605 * @flags: send flags
1606 * @protocol: wire protocol
1607 **/
1608 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
1609 u32 flags, __be16 protocol)
1610 {
1611 struct i40e_filter_program_desc *fdir_desc;
1612 struct i40e_pf *pf = tx_ring->vsi->back;
1613 union {
1614 unsigned char *network;
1615 struct iphdr *ipv4;
1616 struct ipv6hdr *ipv6;
1617 } hdr;
1618 struct tcphdr *th;
1619 unsigned int hlen;
1620 u32 flex_ptype, dtype_cmd;
1621 u16 i;
1622
1623 /* make sure ATR is enabled */
1624 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
1625 return;
1626
1627 /* if sampling is disabled do nothing */
1628 if (!tx_ring->atr_sample_rate)
1629 return;
1630
1631 /* snag network header to get L4 type and address */
1632 hdr.network = skb_network_header(skb);
1633
1634 /* Currently only IPv4/IPv6 with TCP is supported */
1635 if (protocol == htons(ETH_P_IP)) {
1636 if (hdr.ipv4->protocol != IPPROTO_TCP)
1637 return;
1638
1639 /* access ihl as a u8 to avoid unaligned access on ia64 */
1640 hlen = (hdr.network[0] & 0x0F) << 2;
1641 } else if (protocol == htons(ETH_P_IPV6)) {
1642 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1643 return;
1644
1645 hlen = sizeof(struct ipv6hdr);
1646 } else {
1647 return;
1648 }
1649
1650 th = (struct tcphdr *)(hdr.network + hlen);
1651
1652 /* Due to lack of space, no more new filters can be programmed */
1653 if (th->syn && (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
1654 return;
1655
1656 tx_ring->atr_count++;
1657
1658 /* sample on all syn/fin/rst packets or once every atr sample rate */
1659 if (!th->fin &&
1660 !th->syn &&
1661 !th->rst &&
1662 (tx_ring->atr_count < tx_ring->atr_sample_rate))
1663 return;
1664
1665 tx_ring->atr_count = 0;
1666
1667 /* grab the next descriptor */
1668 i = tx_ring->next_to_use;
1669 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
1670
1671 i++;
1672 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1673
1674 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1675 I40E_TXD_FLTR_QW0_QINDEX_MASK;
1676 flex_ptype |= (protocol == htons(ETH_P_IP)) ?
1677 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1678 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1679 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1680 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1681
1682 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1683
1684 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
1685
1686 dtype_cmd |= (th->fin || th->rst) ?
1687 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1688 I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1689 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1690 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1691
1692 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1693 I40E_TXD_FLTR_QW1_DEST_SHIFT;
1694
1695 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1696 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1697
1698 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
1699 dtype_cmd |=
1700 ((u32)pf->fd_atr_cnt_idx << I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
1701 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
1702
1703 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
1704 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
1705 }
1706
1707 /**
1708 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1709 * @skb: send buffer
1710 * @tx_ring: ring to send buffer on
1711 * @flags: the tx flags to be set
1712 *
1713 * Checks the skb and set up correspondingly several generic transmit flags
1714 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1715 *
1716 * Returns error code indicate the frame should be dropped upon error and the
1717 * otherwise returns 0 to indicate the flags has been set properly.
1718 **/
1719 static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
1720 struct i40e_ring *tx_ring,
1721 u32 *flags)
1722 {
1723 __be16 protocol = skb->protocol;
1724 u32 tx_flags = 0;
1725
1726 /* if we have a HW VLAN tag being added, default to the HW one */
1727 if (vlan_tx_tag_present(skb)) {
1728 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1729 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1730 /* else if it is a SW VLAN, check the next protocol and store the tag */
1731 } else if (protocol == htons(ETH_P_8021Q)) {
1732 struct vlan_hdr *vhdr, _vhdr;
1733 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1734 if (!vhdr)
1735 return -EINVAL;
1736
1737 protocol = vhdr->h_vlan_encapsulated_proto;
1738 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1739 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1740 }
1741
1742 /* Insert 802.1p priority into VLAN header */
1743 if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) &&
1744 ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
1745 (skb->priority != TC_PRIO_CONTROL))) {
1746 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
1747 tx_flags |= (skb->priority & 0x7) <<
1748 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
1749 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
1750 struct vlan_ethhdr *vhdr;
1751 int rc;
1752
1753 rc = skb_cow_head(skb, 0);
1754 if (rc < 0)
1755 return rc;
1756 vhdr = (struct vlan_ethhdr *)skb->data;
1757 vhdr->h_vlan_TCI = htons(tx_flags >>
1758 I40E_TX_FLAGS_VLAN_SHIFT);
1759 } else {
1760 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1761 }
1762 }
1763 *flags = tx_flags;
1764 return 0;
1765 }
1766
1767 /**
1768 * i40e_tso - set up the tso context descriptor
1769 * @tx_ring: ptr to the ring to send
1770 * @skb: ptr to the skb we're sending
1771 * @tx_flags: the collected send information
1772 * @protocol: the send protocol
1773 * @hdr_len: ptr to the size of the packet header
1774 * @cd_tunneling: ptr to context descriptor bits
1775 *
1776 * Returns 0 if no TSO can happen, 1 if tso is going, or error
1777 **/
1778 static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
1779 u32 tx_flags, __be16 protocol, u8 *hdr_len,
1780 u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling)
1781 {
1782 u32 cd_cmd, cd_tso_len, cd_mss;
1783 struct ipv6hdr *ipv6h;
1784 struct tcphdr *tcph;
1785 struct iphdr *iph;
1786 u32 l4len;
1787 int err;
1788
1789 if (!skb_is_gso(skb))
1790 return 0;
1791
1792 err = skb_cow_head(skb, 0);
1793 if (err < 0)
1794 return err;
1795
1796 if (protocol == htons(ETH_P_IP)) {
1797 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
1798 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1799 iph->tot_len = 0;
1800 iph->check = 0;
1801 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1802 0, IPPROTO_TCP, 0);
1803 } else if (skb_is_gso_v6(skb)) {
1804
1805 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb)
1806 : ipv6_hdr(skb);
1807 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1808 ipv6h->payload_len = 0;
1809 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
1810 0, IPPROTO_TCP, 0);
1811 }
1812
1813 l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1814 *hdr_len = (skb->encapsulation
1815 ? (skb_inner_transport_header(skb) - skb->data)
1816 : skb_transport_offset(skb)) + l4len;
1817
1818 /* find the field values */
1819 cd_cmd = I40E_TX_CTX_DESC_TSO;
1820 cd_tso_len = skb->len - *hdr_len;
1821 cd_mss = skb_shinfo(skb)->gso_size;
1822 *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1823 ((u64)cd_tso_len <<
1824 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1825 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1826 return 1;
1827 }
1828
1829 /**
1830 * i40e_tsyn - set up the tsyn context descriptor
1831 * @tx_ring: ptr to the ring to send
1832 * @skb: ptr to the skb we're sending
1833 * @tx_flags: the collected send information
1834 *
1835 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
1836 **/
1837 static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
1838 u32 tx_flags, u64 *cd_type_cmd_tso_mss)
1839 {
1840 struct i40e_pf *pf;
1841
1842 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1843 return 0;
1844
1845 /* Tx timestamps cannot be sampled when doing TSO */
1846 if (tx_flags & I40E_TX_FLAGS_TSO)
1847 return 0;
1848
1849 /* only timestamp the outbound packet if the user has requested it and
1850 * we are not already transmitting a packet to be timestamped
1851 */
1852 pf = i40e_netdev_to_pf(tx_ring->netdev);
1853 if (pf->ptp_tx && !pf->ptp_tx_skb) {
1854 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1855 pf->ptp_tx_skb = skb_get(skb);
1856 } else {
1857 return 0;
1858 }
1859
1860 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
1861 I40E_TXD_CTX_QW1_CMD_SHIFT;
1862
1863 return 1;
1864 }
1865
1866 /**
1867 * i40e_tx_enable_csum - Enable Tx checksum offloads
1868 * @skb: send buffer
1869 * @tx_flags: Tx flags currently set
1870 * @td_cmd: Tx descriptor command bits to set
1871 * @td_offset: Tx descriptor header offsets to set
1872 * @cd_tunneling: ptr to context desc bits
1873 **/
1874 static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags,
1875 u32 *td_cmd, u32 *td_offset,
1876 struct i40e_ring *tx_ring,
1877 u32 *cd_tunneling)
1878 {
1879 struct ipv6hdr *this_ipv6_hdr;
1880 unsigned int this_tcp_hdrlen;
1881 struct iphdr *this_ip_hdr;
1882 u32 network_hdr_len;
1883 u8 l4_hdr = 0;
1884
1885 if (skb->encapsulation) {
1886 network_hdr_len = skb_inner_network_header_len(skb);
1887 this_ip_hdr = inner_ip_hdr(skb);
1888 this_ipv6_hdr = inner_ipv6_hdr(skb);
1889 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
1890
1891 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1892
1893 if (tx_flags & I40E_TX_FLAGS_TSO) {
1894 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
1895 ip_hdr(skb)->check = 0;
1896 } else {
1897 *cd_tunneling |=
1898 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1899 }
1900 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1901 if (tx_flags & I40E_TX_FLAGS_TSO) {
1902 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
1903 ip_hdr(skb)->check = 0;
1904 } else {
1905 *cd_tunneling |=
1906 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1907 }
1908 }
1909
1910 /* Now set the ctx descriptor fields */
1911 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
1912 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
1913 I40E_TXD_CTX_UDP_TUNNELING |
1914 ((skb_inner_network_offset(skb) -
1915 skb_transport_offset(skb)) >> 1) <<
1916 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1917
1918 } else {
1919 network_hdr_len = skb_network_header_len(skb);
1920 this_ip_hdr = ip_hdr(skb);
1921 this_ipv6_hdr = ipv6_hdr(skb);
1922 this_tcp_hdrlen = tcp_hdrlen(skb);
1923 }
1924
1925 /* Enable IP checksum offloads */
1926 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1927 l4_hdr = this_ip_hdr->protocol;
1928 /* the stack computes the IP header already, the only time we
1929 * need the hardware to recompute it is in the case of TSO.
1930 */
1931 if (tx_flags & I40E_TX_FLAGS_TSO) {
1932 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
1933 this_ip_hdr->check = 0;
1934 } else {
1935 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
1936 }
1937 /* Now set the td_offset for IP header length */
1938 *td_offset = (network_hdr_len >> 2) <<
1939 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1940 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1941 l4_hdr = this_ipv6_hdr->nexthdr;
1942 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1943 /* Now set the td_offset for IP header length */
1944 *td_offset = (network_hdr_len >> 2) <<
1945 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1946 }
1947 /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
1948 *td_offset |= (skb_network_offset(skb) >> 1) <<
1949 I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1950
1951 /* Enable L4 checksum offloads */
1952 switch (l4_hdr) {
1953 case IPPROTO_TCP:
1954 /* enable checksum offloads */
1955 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1956 *td_offset |= (this_tcp_hdrlen >> 2) <<
1957 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1958 break;
1959 case IPPROTO_SCTP:
1960 /* enable SCTP checksum offload */
1961 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1962 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
1963 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1964 break;
1965 case IPPROTO_UDP:
1966 /* enable UDP checksum offload */
1967 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1968 *td_offset |= (sizeof(struct udphdr) >> 2) <<
1969 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1970 break;
1971 default:
1972 break;
1973 }
1974 }
1975
1976 /**
1977 * i40e_create_tx_ctx Build the Tx context descriptor
1978 * @tx_ring: ring to create the descriptor on
1979 * @cd_type_cmd_tso_mss: Quad Word 1
1980 * @cd_tunneling: Quad Word 0 - bits 0-31
1981 * @cd_l2tag2: Quad Word 0 - bits 32-63
1982 **/
1983 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1984 const u64 cd_type_cmd_tso_mss,
1985 const u32 cd_tunneling, const u32 cd_l2tag2)
1986 {
1987 struct i40e_tx_context_desc *context_desc;
1988 int i = tx_ring->next_to_use;
1989
1990 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
1991 !cd_tunneling && !cd_l2tag2)
1992 return;
1993
1994 /* grab the next descriptor */
1995 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
1996
1997 i++;
1998 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1999
2000 /* cpu_to_le32 and assign to struct fields */
2001 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
2002 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
2003 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
2004 }
2005
2006 /**
2007 * i40e_tx_map - Build the Tx descriptor
2008 * @tx_ring: ring to send buffer on
2009 * @skb: send buffer
2010 * @first: first buffer info buffer to use
2011 * @tx_flags: collected send information
2012 * @hdr_len: size of the packet header
2013 * @td_cmd: the command field in the descriptor
2014 * @td_offset: offset for checksum or crc
2015 **/
2016 static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
2017 struct i40e_tx_buffer *first, u32 tx_flags,
2018 const u8 hdr_len, u32 td_cmd, u32 td_offset)
2019 {
2020 unsigned int data_len = skb->data_len;
2021 unsigned int size = skb_headlen(skb);
2022 struct skb_frag_struct *frag;
2023 struct i40e_tx_buffer *tx_bi;
2024 struct i40e_tx_desc *tx_desc;
2025 u16 i = tx_ring->next_to_use;
2026 u32 td_tag = 0;
2027 dma_addr_t dma;
2028 u16 gso_segs;
2029
2030 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
2031 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
2032 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
2033 I40E_TX_FLAGS_VLAN_SHIFT;
2034 }
2035
2036 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
2037 gso_segs = skb_shinfo(skb)->gso_segs;
2038 else
2039 gso_segs = 1;
2040
2041 /* multiply data chunks by size of headers */
2042 first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
2043 first->gso_segs = gso_segs;
2044 first->skb = skb;
2045 first->tx_flags = tx_flags;
2046
2047 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2048
2049 tx_desc = I40E_TX_DESC(tx_ring, i);
2050 tx_bi = first;
2051
2052 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2053 if (dma_mapping_error(tx_ring->dev, dma))
2054 goto dma_error;
2055
2056 /* record length, and DMA address */
2057 dma_unmap_len_set(tx_bi, len, size);
2058 dma_unmap_addr_set(tx_bi, dma, dma);
2059
2060 tx_desc->buffer_addr = cpu_to_le64(dma);
2061
2062 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
2063 tx_desc->cmd_type_offset_bsz =
2064 build_ctob(td_cmd, td_offset,
2065 I40E_MAX_DATA_PER_TXD, td_tag);
2066
2067 tx_desc++;
2068 i++;
2069 if (i == tx_ring->count) {
2070 tx_desc = I40E_TX_DESC(tx_ring, 0);
2071 i = 0;
2072 }
2073
2074 dma += I40E_MAX_DATA_PER_TXD;
2075 size -= I40E_MAX_DATA_PER_TXD;
2076
2077 tx_desc->buffer_addr = cpu_to_le64(dma);
2078 }
2079
2080 if (likely(!data_len))
2081 break;
2082
2083 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2084 size, td_tag);
2085
2086 tx_desc++;
2087 i++;
2088 if (i == tx_ring->count) {
2089 tx_desc = I40E_TX_DESC(tx_ring, 0);
2090 i = 0;
2091 }
2092
2093 size = skb_frag_size(frag);
2094 data_len -= size;
2095
2096 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2097 DMA_TO_DEVICE);
2098
2099 tx_bi = &tx_ring->tx_bi[i];
2100 }
2101
2102 /* Place RS bit on last descriptor of any packet that spans across the
2103 * 4th descriptor (WB_STRIDE aka 0x3) in a 64B cacheline.
2104 */
2105 #define WB_STRIDE 0x3
2106 if (((i & WB_STRIDE) != WB_STRIDE) &&
2107 (first <= &tx_ring->tx_bi[i]) &&
2108 (first >= &tx_ring->tx_bi[i & ~WB_STRIDE])) {
2109 tx_desc->cmd_type_offset_bsz =
2110 build_ctob(td_cmd, td_offset, size, td_tag) |
2111 cpu_to_le64((u64)I40E_TX_DESC_CMD_EOP <<
2112 I40E_TXD_QW1_CMD_SHIFT);
2113 } else {
2114 tx_desc->cmd_type_offset_bsz =
2115 build_ctob(td_cmd, td_offset, size, td_tag) |
2116 cpu_to_le64((u64)I40E_TXD_CMD <<
2117 I40E_TXD_QW1_CMD_SHIFT);
2118 }
2119
2120 netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
2121 tx_ring->queue_index),
2122 first->bytecount);
2123
2124 /* set the timestamp */
2125 first->time_stamp = jiffies;
2126
2127 /* Force memory writes to complete before letting h/w
2128 * know there are new descriptors to fetch. (Only
2129 * applicable for weak-ordered memory model archs,
2130 * such as IA-64).
2131 */
2132 wmb();
2133
2134 /* set next_to_watch value indicating a packet is present */
2135 first->next_to_watch = tx_desc;
2136
2137 i++;
2138 if (i == tx_ring->count)
2139 i = 0;
2140
2141 tx_ring->next_to_use = i;
2142
2143 /* notify HW of packet */
2144 writel(i, tx_ring->tail);
2145
2146 return;
2147
2148 dma_error:
2149 dev_info(tx_ring->dev, "TX DMA map failed\n");
2150
2151 /* clear dma mappings for failed tx_bi map */
2152 for (;;) {
2153 tx_bi = &tx_ring->tx_bi[i];
2154 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
2155 if (tx_bi == first)
2156 break;
2157 if (i == 0)
2158 i = tx_ring->count;
2159 i--;
2160 }
2161
2162 tx_ring->next_to_use = i;
2163 }
2164
2165 /**
2166 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
2167 * @tx_ring: the ring to be checked
2168 * @size: the size buffer we want to assure is available
2169 *
2170 * Returns -EBUSY if a stop is needed, else 0
2171 **/
2172 static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2173 {
2174 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2175 /* Memory barrier before checking head and tail */
2176 smp_mb();
2177
2178 /* Check again in a case another CPU has just made room available. */
2179 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2180 return -EBUSY;
2181
2182 /* A reprieve! - use start_queue because it doesn't call schedule */
2183 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2184 ++tx_ring->tx_stats.restart_queue;
2185 return 0;
2186 }
2187
2188 /**
2189 * i40e_maybe_stop_tx - 1st level check for tx stop conditions
2190 * @tx_ring: the ring to be checked
2191 * @size: the size buffer we want to assure is available
2192 *
2193 * Returns 0 if stop is not needed
2194 **/
2195 static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2196 {
2197 if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
2198 return 0;
2199 return __i40e_maybe_stop_tx(tx_ring, size);
2200 }
2201
2202 /**
2203 * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
2204 * @skb: send buffer
2205 * @tx_ring: ring to send buffer on
2206 *
2207 * Returns number of data descriptors needed for this skb. Returns 0 to indicate
2208 * there is not enough descriptors available in this ring since we need at least
2209 * one descriptor.
2210 **/
2211 static int i40e_xmit_descriptor_count(struct sk_buff *skb,
2212 struct i40e_ring *tx_ring)
2213 {
2214 unsigned int f;
2215 int count = 0;
2216
2217 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2218 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
2219 * + 4 desc gap to avoid the cache line where head is,
2220 * + 1 desc for context descriptor,
2221 * otherwise try next time
2222 */
2223 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2224 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2225
2226 count += TXD_USE_COUNT(skb_headlen(skb));
2227 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
2228 tx_ring->tx_stats.tx_busy++;
2229 return 0;
2230 }
2231 return count;
2232 }
2233
2234 /**
2235 * i40e_xmit_frame_ring - Sends buffer on Tx ring
2236 * @skb: send buffer
2237 * @tx_ring: ring to send buffer on
2238 *
2239 * Returns NETDEV_TX_OK if sent, else an error code
2240 **/
2241 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2242 struct i40e_ring *tx_ring)
2243 {
2244 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2245 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2246 struct i40e_tx_buffer *first;
2247 u32 td_offset = 0;
2248 u32 tx_flags = 0;
2249 __be16 protocol;
2250 u32 td_cmd = 0;
2251 u8 hdr_len = 0;
2252 int tsyn;
2253 int tso;
2254 if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
2255 return NETDEV_TX_BUSY;
2256
2257 /* prepare the xmit flags */
2258 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2259 goto out_drop;
2260
2261 /* obtain protocol of skb */
2262 protocol = skb->protocol;
2263
2264 /* record the location of the first descriptor for this packet */
2265 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2266
2267 /* setup IPv4/IPv6 offloads */
2268 if (protocol == htons(ETH_P_IP))
2269 tx_flags |= I40E_TX_FLAGS_IPV4;
2270 else if (protocol == htons(ETH_P_IPV6))
2271 tx_flags |= I40E_TX_FLAGS_IPV6;
2272
2273 tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len,
2274 &cd_type_cmd_tso_mss, &cd_tunneling);
2275
2276 if (tso < 0)
2277 goto out_drop;
2278 else if (tso)
2279 tx_flags |= I40E_TX_FLAGS_TSO;
2280
2281 skb_tx_timestamp(skb);
2282
2283 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
2284
2285 if (tsyn)
2286 tx_flags |= I40E_TX_FLAGS_TSYN;
2287
2288 /* always enable CRC insertion offload */
2289 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2290
2291 /* Always offload the checksum, since it's in the data descriptor */
2292 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2293 tx_flags |= I40E_TX_FLAGS_CSUM;
2294
2295 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset,
2296 tx_ring, &cd_tunneling);
2297 }
2298
2299 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2300 cd_tunneling, cd_l2tag2);
2301
2302 /* Add Flow Director ATR if it's enabled.
2303 *
2304 * NOTE: this must always be directly before the data descriptor.
2305 */
2306 i40e_atr(tx_ring, skb, tx_flags, protocol);
2307
2308 i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2309 td_cmd, td_offset);
2310
2311 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2312
2313 return NETDEV_TX_OK;
2314
2315 out_drop:
2316 dev_kfree_skb_any(skb);
2317 return NETDEV_TX_OK;
2318 }
2319
2320 /**
2321 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2322 * @skb: send buffer
2323 * @netdev: network interface device structure
2324 *
2325 * Returns NETDEV_TX_OK if sent, else an error code
2326 **/
2327 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2328 {
2329 struct i40e_netdev_priv *np = netdev_priv(netdev);
2330 struct i40e_vsi *vsi = np->vsi;
2331 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
2332
2333 /* hardware can't handle really short frames, hardware padding works
2334 * beyond this point
2335 */
2336 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2337 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2338 return NETDEV_TX_OK;
2339 skb->len = I40E_MIN_TX_LEN;
2340 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2341 }
2342
2343 return i40e_xmit_frame_ring(skb, tx_ring);
2344 }
Linux kernel - Deliverables
This page took 0.11235 seconds and 6 git commands to generate.