2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/mlx4/cq.h>
36 #include <linux/slab.h>
37 #include <linux/mlx4/qp.h>
38 #include <linux/skbuff.h>
39 #include <linux/if_vlan.h>
40 #include <linux/prefetch.h>
41 #include <linux/vmalloc.h>
42 #include <linux/tcp.h>
44 #include <linux/moduleparam.h>
48 int mlx4_en_create_tx_ring(struct mlx4_en_priv
*priv
,
49 struct mlx4_en_tx_ring
**pring
, u32 size
,
50 u16 stride
, int node
, int queue_index
)
52 struct mlx4_en_dev
*mdev
= priv
->mdev
;
53 struct mlx4_en_tx_ring
*ring
;
57 ring
= kzalloc_node(sizeof(*ring
), GFP_KERNEL
, node
);
59 ring
= kzalloc(sizeof(*ring
), GFP_KERNEL
);
61 en_err(priv
, "Failed allocating TX ring\n");
67 ring
->size_mask
= size
- 1;
68 ring
->stride
= stride
;
70 tmp
= size
* sizeof(struct mlx4_en_tx_info
);
71 ring
->tx_info
= kmalloc_node(tmp
, GFP_KERNEL
| __GFP_NOWARN
, node
);
73 ring
->tx_info
= vmalloc(tmp
);
80 en_dbg(DRV
, priv
, "Allocated tx_info ring at addr:%p size:%d\n",
83 ring
->bounce_buf
= kmalloc_node(MAX_DESC_SIZE
, GFP_KERNEL
, node
);
84 if (!ring
->bounce_buf
) {
85 ring
->bounce_buf
= kmalloc(MAX_DESC_SIZE
, GFP_KERNEL
);
86 if (!ring
->bounce_buf
) {
91 ring
->buf_size
= ALIGN(size
* ring
->stride
, MLX4_EN_PAGE_SIZE
);
93 /* Allocate HW buffers on provided NUMA node */
94 set_dev_node(&mdev
->dev
->pdev
->dev
, node
);
95 err
= mlx4_alloc_hwq_res(mdev
->dev
, &ring
->wqres
, ring
->buf_size
,
97 set_dev_node(&mdev
->dev
->pdev
->dev
, mdev
->dev
->numa_node
);
99 en_err(priv
, "Failed allocating hwq resources\n");
103 err
= mlx4_en_map_buffer(&ring
->wqres
.buf
);
105 en_err(priv
, "Failed to map TX buffer\n");
109 ring
->buf
= ring
->wqres
.buf
.direct
.buf
;
111 en_dbg(DRV
, priv
, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n",
112 ring
, ring
->buf
, ring
->size
, ring
->buf_size
,
113 (unsigned long long) ring
->wqres
.buf
.direct
.map
);
115 err
= mlx4_qp_reserve_range(mdev
->dev
, 1, 1, &ring
->qpn
,
116 MLX4_RESERVE_ETH_BF_QP
);
118 en_err(priv
, "failed reserving qp for TX ring\n");
122 err
= mlx4_qp_alloc(mdev
->dev
, ring
->qpn
, &ring
->qp
, GFP_KERNEL
);
124 en_err(priv
, "Failed allocating qp %d\n", ring
->qpn
);
127 ring
->qp
.event
= mlx4_en_sqp_event
;
129 err
= mlx4_bf_alloc(mdev
->dev
, &ring
->bf
, node
);
131 en_dbg(DRV
, priv
, "working without blueflame (%d)\n", err
);
132 ring
->bf
.uar
= &mdev
->priv_uar
;
133 ring
->bf
.uar
->map
= mdev
->uar_map
;
134 ring
->bf_enabled
= false;
135 ring
->bf_alloced
= false;
136 priv
->pflags
&= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME
;
138 ring
->bf_alloced
= true;
139 ring
->bf_enabled
= !!(priv
->pflags
&
140 MLX4_EN_PRIV_FLAGS_BLUEFLAME
);
143 ring
->hwtstamp_tx_type
= priv
->hwtstamp_config
.tx_type
;
144 ring
->queue_index
= queue_index
;
146 if (queue_index
< priv
->num_tx_rings_p_up
&& cpu_online(queue_index
))
147 cpumask_set_cpu(queue_index
, &ring
->affinity_mask
);
153 mlx4_qp_release_range(mdev
->dev
, ring
->qpn
, 1);
155 mlx4_en_unmap_buffer(&ring
->wqres
.buf
);
157 mlx4_free_hwq_res(mdev
->dev
, &ring
->wqres
, ring
->buf_size
);
159 kfree(ring
->bounce_buf
);
160 ring
->bounce_buf
= NULL
;
162 kvfree(ring
->tx_info
);
163 ring
->tx_info
= NULL
;
170 void mlx4_en_destroy_tx_ring(struct mlx4_en_priv
*priv
,
171 struct mlx4_en_tx_ring
**pring
)
173 struct mlx4_en_dev
*mdev
= priv
->mdev
;
174 struct mlx4_en_tx_ring
*ring
= *pring
;
175 en_dbg(DRV
, priv
, "Destroying tx ring, qpn: %d\n", ring
->qpn
);
177 if (ring
->bf_alloced
)
178 mlx4_bf_free(mdev
->dev
, &ring
->bf
);
179 mlx4_qp_remove(mdev
->dev
, &ring
->qp
);
180 mlx4_qp_free(mdev
->dev
, &ring
->qp
);
181 mlx4_en_unmap_buffer(&ring
->wqres
.buf
);
182 mlx4_free_hwq_res(mdev
->dev
, &ring
->wqres
, ring
->buf_size
);
183 kfree(ring
->bounce_buf
);
184 ring
->bounce_buf
= NULL
;
185 kvfree(ring
->tx_info
);
186 ring
->tx_info
= NULL
;
191 int mlx4_en_activate_tx_ring(struct mlx4_en_priv
*priv
,
192 struct mlx4_en_tx_ring
*ring
,
193 int cq
, int user_prio
)
195 struct mlx4_en_dev
*mdev
= priv
->mdev
;
200 ring
->cons
= 0xffffffff;
201 ring
->last_nr_txbb
= 1;
202 memset(ring
->tx_info
, 0, ring
->size
* sizeof(struct mlx4_en_tx_info
));
203 memset(ring
->buf
, 0, ring
->buf_size
);
205 ring
->qp_state
= MLX4_QP_STATE_RST
;
206 ring
->doorbell_qpn
= cpu_to_be32(ring
->qp
.qpn
<< 8);
207 ring
->mr_key
= cpu_to_be32(mdev
->mr
.key
);
209 mlx4_en_fill_qp_context(priv
, ring
->size
, ring
->stride
, 1, 0, ring
->qpn
,
210 ring
->cqn
, user_prio
, &ring
->context
);
211 if (ring
->bf_alloced
)
212 ring
->context
.usr_page
= cpu_to_be32(ring
->bf
.uar
->index
);
214 err
= mlx4_qp_to_ready(mdev
->dev
, &ring
->wqres
.mtt
, &ring
->context
,
215 &ring
->qp
, &ring
->qp_state
);
216 if (!user_prio
&& cpu_online(ring
->queue_index
))
217 netif_set_xps_queue(priv
->dev
, &ring
->affinity_mask
,
223 void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv
*priv
,
224 struct mlx4_en_tx_ring
*ring
)
226 struct mlx4_en_dev
*mdev
= priv
->mdev
;
228 mlx4_qp_modify(mdev
->dev
, NULL
, ring
->qp_state
,
229 MLX4_QP_STATE_RST
, NULL
, 0, 0, &ring
->qp
);
232 static void mlx4_en_stamp_wqe(struct mlx4_en_priv
*priv
,
233 struct mlx4_en_tx_ring
*ring
, int index
,
236 __be32 stamp
= cpu_to_be32(STAMP_VAL
| (!!owner
<< STAMP_SHIFT
));
237 struct mlx4_en_tx_desc
*tx_desc
= ring
->buf
+ index
* TXBB_SIZE
;
238 struct mlx4_en_tx_info
*tx_info
= &ring
->tx_info
[index
];
239 void *end
= ring
->buf
+ ring
->buf_size
;
240 __be32
*ptr
= (__be32
*)tx_desc
;
243 /* Optimize the common case when there are no wraparounds */
244 if (likely((void *)tx_desc
+ tx_info
->nr_txbb
* TXBB_SIZE
<= end
)) {
245 /* Stamp the freed descriptor */
246 for (i
= 0; i
< tx_info
->nr_txbb
* TXBB_SIZE
;
252 /* Stamp the freed descriptor */
253 for (i
= 0; i
< tx_info
->nr_txbb
* TXBB_SIZE
;
257 if ((void *)ptr
>= end
) {
259 stamp
^= cpu_to_be32(0x80000000);
266 static u32
mlx4_en_free_tx_desc(struct mlx4_en_priv
*priv
,
267 struct mlx4_en_tx_ring
*ring
,
268 int index
, u8 owner
, u64 timestamp
)
270 struct mlx4_en_tx_info
*tx_info
= &ring
->tx_info
[index
];
271 struct mlx4_en_tx_desc
*tx_desc
= ring
->buf
+ index
* TXBB_SIZE
;
272 struct mlx4_wqe_data_seg
*data
= (void *) tx_desc
+ tx_info
->data_offset
;
273 void *end
= ring
->buf
+ ring
->buf_size
;
274 struct sk_buff
*skb
= tx_info
->skb
;
275 int nr_maps
= tx_info
->nr_maps
;
278 /* We do not touch skb here, so prefetch skb->users location
279 * to speedup consume_skb()
281 prefetchw(&skb
->users
);
283 if (unlikely(timestamp
)) {
284 struct skb_shared_hwtstamps hwts
;
286 mlx4_en_fill_hwtstamps(priv
->mdev
, &hwts
, timestamp
);
287 skb_tstamp_tx(skb
, &hwts
);
290 /* Optimize the common case when there are no wraparounds */
291 if (likely((void *) tx_desc
+ tx_info
->nr_txbb
* TXBB_SIZE
<= end
)) {
294 dma_unmap_single(priv
->ddev
,
296 tx_info
->map0_byte_count
,
299 dma_unmap_page(priv
->ddev
,
301 tx_info
->map0_byte_count
,
303 for (i
= 1; i
< nr_maps
; i
++) {
305 dma_unmap_page(priv
->ddev
,
306 (dma_addr_t
)be64_to_cpu(data
->addr
),
307 be32_to_cpu(data
->byte_count
),
313 if ((void *) data
>= end
) {
314 data
= ring
->buf
+ ((void *)data
- end
);
318 dma_unmap_single(priv
->ddev
,
320 tx_info
->map0_byte_count
,
323 dma_unmap_page(priv
->ddev
,
325 tx_info
->map0_byte_count
,
327 for (i
= 1; i
< nr_maps
; i
++) {
329 /* Check for wraparound before unmapping */
330 if ((void *) data
>= end
)
332 dma_unmap_page(priv
->ddev
,
333 (dma_addr_t
)be64_to_cpu(data
->addr
),
334 be32_to_cpu(data
->byte_count
),
339 dev_consume_skb_any(skb
);
340 return tx_info
->nr_txbb
;
344 int mlx4_en_free_tx_buf(struct net_device
*dev
, struct mlx4_en_tx_ring
*ring
)
346 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
349 /* Skip last polled descriptor */
350 ring
->cons
+= ring
->last_nr_txbb
;
351 en_dbg(DRV
, priv
, "Freeing Tx buf - cons:0x%x prod:0x%x\n",
352 ring
->cons
, ring
->prod
);
354 if ((u32
) (ring
->prod
- ring
->cons
) > ring
->size
) {
355 if (netif_msg_tx_err(priv
))
356 en_warn(priv
, "Tx consumer passed producer!\n");
360 while (ring
->cons
!= ring
->prod
) {
361 ring
->last_nr_txbb
= mlx4_en_free_tx_desc(priv
, ring
,
362 ring
->cons
& ring
->size_mask
,
363 !!(ring
->cons
& ring
->size
), 0);
364 ring
->cons
+= ring
->last_nr_txbb
;
368 netdev_tx_reset_queue(ring
->tx_queue
);
371 en_dbg(DRV
, priv
, "Freed %d uncompleted tx descriptors\n", cnt
);
376 static bool mlx4_en_process_tx_cq(struct net_device
*dev
,
377 struct mlx4_en_cq
*cq
)
379 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
380 struct mlx4_cq
*mcq
= &cq
->mcq
;
381 struct mlx4_en_tx_ring
*ring
= priv
->tx_ring
[cq
->ring
];
382 struct mlx4_cqe
*cqe
;
384 u16 new_index
, ring_index
, stamp_index
;
385 u32 txbbs_skipped
= 0;
387 u32 cons_index
= mcq
->cons_index
;
389 u32 size_mask
= ring
->size_mask
;
390 struct mlx4_cqe
*buf
= cq
->buf
;
393 int factor
= priv
->cqe_factor
;
396 int budget
= priv
->tx_work_limit
;
403 netdev_txq_bql_complete_prefetchw(ring
->tx_queue
);
405 index
= cons_index
& size_mask
;
406 cqe
= mlx4_en_get_cqe(buf
, index
, priv
->cqe_size
) + factor
;
407 last_nr_txbb
= ACCESS_ONCE(ring
->last_nr_txbb
);
408 ring_cons
= ACCESS_ONCE(ring
->cons
);
409 ring_index
= ring_cons
& size_mask
;
410 stamp_index
= ring_index
;
412 /* Process all completed CQEs */
413 while (XNOR(cqe
->owner_sr_opcode
& MLX4_CQE_OWNER_MASK
,
414 cons_index
& size
) && (done
< budget
)) {
416 * make sure we read the CQE after we read the
421 if (unlikely((cqe
->owner_sr_opcode
& MLX4_CQE_OPCODE_MASK
) ==
422 MLX4_CQE_OPCODE_ERROR
)) {
423 struct mlx4_err_cqe
*cqe_err
= (struct mlx4_err_cqe
*)cqe
;
425 en_err(priv
, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
426 cqe_err
->vendor_err_syndrome
,
430 /* Skip over last polled CQE */
431 new_index
= be16_to_cpu(cqe
->wqe_index
) & size_mask
;
434 txbbs_skipped
+= last_nr_txbb
;
435 ring_index
= (ring_index
+ last_nr_txbb
) & size_mask
;
436 if (ring
->tx_info
[ring_index
].ts_requested
)
437 timestamp
= mlx4_en_get_cqe_ts(cqe
);
439 /* free next descriptor */
440 last_nr_txbb
= mlx4_en_free_tx_desc(
441 priv
, ring
, ring_index
,
442 !!((ring_cons
+ txbbs_skipped
) &
443 ring
->size
), timestamp
);
445 mlx4_en_stamp_wqe(priv
, ring
, stamp_index
,
446 !!((ring_cons
+ txbbs_stamp
) &
448 stamp_index
= ring_index
;
449 txbbs_stamp
= txbbs_skipped
;
451 bytes
+= ring
->tx_info
[ring_index
].nr_bytes
;
452 } while ((++done
< budget
) && (ring_index
!= new_index
));
455 index
= cons_index
& size_mask
;
456 cqe
= mlx4_en_get_cqe(buf
, index
, priv
->cqe_size
) + factor
;
461 * To prevent CQ overflow we first update CQ consumer and only then
464 mcq
->cons_index
= cons_index
;
468 /* we want to dirty this cache line once */
469 ACCESS_ONCE(ring
->last_nr_txbb
) = last_nr_txbb
;
470 ACCESS_ONCE(ring
->cons
) = ring_cons
+ txbbs_skipped
;
472 netdev_tx_completed_queue(ring
->tx_queue
, packets
, bytes
);
475 * Wakeup Tx queue if this stopped, and at least 1 packet
478 if (netif_tx_queue_stopped(ring
->tx_queue
) && txbbs_skipped
> 0) {
479 netif_tx_wake_queue(ring
->tx_queue
);
482 return done
< budget
;
485 void mlx4_en_tx_irq(struct mlx4_cq
*mcq
)
487 struct mlx4_en_cq
*cq
= container_of(mcq
, struct mlx4_en_cq
, mcq
);
488 struct mlx4_en_priv
*priv
= netdev_priv(cq
->dev
);
490 if (likely(priv
->port_up
))
491 napi_schedule_irqoff(&cq
->napi
);
493 mlx4_en_arm_cq(priv
, cq
);
496 /* TX CQ polling - called by NAPI */
497 int mlx4_en_poll_tx_cq(struct napi_struct
*napi
, int budget
)
499 struct mlx4_en_cq
*cq
= container_of(napi
, struct mlx4_en_cq
, napi
);
500 struct net_device
*dev
= cq
->dev
;
501 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
504 clean_complete
= mlx4_en_process_tx_cq(dev
, cq
);
509 mlx4_en_arm_cq(priv
, cq
);
514 static struct mlx4_en_tx_desc
*mlx4_en_bounce_to_desc(struct mlx4_en_priv
*priv
,
515 struct mlx4_en_tx_ring
*ring
,
517 unsigned int desc_size
)
519 u32 copy
= (ring
->size
- index
) * TXBB_SIZE
;
522 for (i
= desc_size
- copy
- 4; i
>= 0; i
-= 4) {
523 if ((i
& (TXBB_SIZE
- 1)) == 0)
526 *((u32
*) (ring
->buf
+ i
)) =
527 *((u32
*) (ring
->bounce_buf
+ copy
+ i
));
530 for (i
= copy
- 4; i
>= 4 ; i
-= 4) {
531 if ((i
& (TXBB_SIZE
- 1)) == 0)
534 *((u32
*) (ring
->buf
+ index
* TXBB_SIZE
+ i
)) =
535 *((u32
*) (ring
->bounce_buf
+ i
));
538 /* Return real descriptor location */
539 return ring
->buf
+ index
* TXBB_SIZE
;
542 /* Decide if skb can be inlined in tx descriptor to avoid dma mapping
544 * It seems strange we do not simply use skb_copy_bits().
545 * This would allow to inline all skbs iff skb->len <= inline_thold
547 * Note that caller already checked skb was not a gso packet
549 static bool is_inline(int inline_thold
, const struct sk_buff
*skb
,
550 const struct skb_shared_info
*shinfo
,
555 if (skb
->len
> inline_thold
|| !inline_thold
)
558 if (shinfo
->nr_frags
== 1) {
559 ptr
= skb_frag_address_safe(&shinfo
->frags
[0]);
565 if (shinfo
->nr_frags
)
570 static int inline_size(const struct sk_buff
*skb
)
572 if (skb
->len
+ CTRL_SIZE
+ sizeof(struct mlx4_wqe_inline_seg
)
573 <= MLX4_INLINE_ALIGN
)
574 return ALIGN(skb
->len
+ CTRL_SIZE
+
575 sizeof(struct mlx4_wqe_inline_seg
), 16);
577 return ALIGN(skb
->len
+ CTRL_SIZE
+ 2 *
578 sizeof(struct mlx4_wqe_inline_seg
), 16);
581 static int get_real_size(const struct sk_buff
*skb
,
582 const struct skb_shared_info
*shinfo
,
583 struct net_device
*dev
,
584 int *lso_header_size
,
588 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
591 if (shinfo
->gso_size
) {
593 if (skb
->encapsulation
)
594 *lso_header_size
= (skb_inner_transport_header(skb
) - skb
->data
) + inner_tcp_hdrlen(skb
);
596 *lso_header_size
= skb_transport_offset(skb
) + tcp_hdrlen(skb
);
597 real_size
= CTRL_SIZE
+ shinfo
->nr_frags
* DS_SIZE
+
598 ALIGN(*lso_header_size
+ 4, DS_SIZE
);
599 if (unlikely(*lso_header_size
!= skb_headlen(skb
))) {
600 /* We add a segment for the skb linear buffer only if
601 * it contains data */
602 if (*lso_header_size
< skb_headlen(skb
))
603 real_size
+= DS_SIZE
;
605 if (netif_msg_tx_err(priv
))
606 en_warn(priv
, "Non-linear headers\n");
611 *lso_header_size
= 0;
612 *inline_ok
= is_inline(priv
->prof
->inline_thold
, skb
,
616 real_size
= inline_size(skb
);
618 real_size
= CTRL_SIZE
+
619 (shinfo
->nr_frags
+ 1) * DS_SIZE
;
625 static void build_inline_wqe(struct mlx4_en_tx_desc
*tx_desc
,
626 const struct sk_buff
*skb
,
627 const struct skb_shared_info
*shinfo
,
628 int real_size
, u16
*vlan_tag
,
629 int tx_ind
, void *fragptr
)
631 struct mlx4_wqe_inline_seg
*inl
= &tx_desc
->inl
;
632 int spc
= MLX4_INLINE_ALIGN
- CTRL_SIZE
- sizeof *inl
;
633 unsigned int hlen
= skb_headlen(skb
);
635 if (skb
->len
<= spc
) {
636 if (likely(skb
->len
>= MIN_PKT_LEN
)) {
637 inl
->byte_count
= cpu_to_be32(1 << 31 | skb
->len
);
639 inl
->byte_count
= cpu_to_be32(1 << 31 | MIN_PKT_LEN
);
640 memset(((void *)(inl
+ 1)) + skb
->len
, 0,
641 MIN_PKT_LEN
- skb
->len
);
643 skb_copy_from_linear_data(skb
, inl
+ 1, hlen
);
644 if (shinfo
->nr_frags
)
645 memcpy(((void *)(inl
+ 1)) + hlen
, fragptr
,
646 skb_frag_size(&shinfo
->frags
[0]));
649 inl
->byte_count
= cpu_to_be32(1 << 31 | spc
);
651 skb_copy_from_linear_data(skb
, inl
+ 1, hlen
);
653 memcpy(((void *)(inl
+ 1)) + hlen
,
654 fragptr
, spc
- hlen
);
655 fragptr
+= spc
- hlen
;
657 inl
= (void *) (inl
+ 1) + spc
;
658 memcpy(((void *)(inl
+ 1)), fragptr
, skb
->len
- spc
);
660 skb_copy_from_linear_data(skb
, inl
+ 1, spc
);
661 inl
= (void *) (inl
+ 1) + spc
;
662 skb_copy_from_linear_data_offset(skb
, spc
, inl
+ 1,
664 if (shinfo
->nr_frags
)
665 memcpy(((void *)(inl
+ 1)) + hlen
- spc
,
667 skb_frag_size(&shinfo
->frags
[0]));
671 inl
->byte_count
= cpu_to_be32(1 << 31 | (skb
->len
- spc
));
675 u16
mlx4_en_select_queue(struct net_device
*dev
, struct sk_buff
*skb
,
676 void *accel_priv
, select_queue_fallback_t fallback
)
678 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
679 u16 rings_p_up
= priv
->num_tx_rings_p_up
;
683 return skb_tx_hash(dev
, skb
);
685 if (vlan_tx_tag_present(skb
))
686 up
= vlan_tx_tag_get(skb
) >> VLAN_PRIO_SHIFT
;
688 return fallback(dev
, skb
) % rings_p_up
+ up
* rings_p_up
;
691 static void mlx4_bf_copy(void __iomem
*dst
, const void *src
,
692 unsigned int bytecnt
)
694 __iowrite64_copy(dst
, src
, bytecnt
/ 8);
697 netdev_tx_t
mlx4_en_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
699 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
700 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
701 struct device
*ddev
= priv
->ddev
;
702 struct mlx4_en_tx_ring
*ring
;
703 struct mlx4_en_tx_desc
*tx_desc
;
704 struct mlx4_wqe_data_seg
*data
;
705 struct mlx4_en_tx_info
*tx_info
;
715 void *fragptr
= NULL
;
725 tx_ind
= skb_get_queue_mapping(skb
);
726 ring
= priv
->tx_ring
[tx_ind
];
728 /* fetch ring->cons far ahead before needing it to avoid stall */
729 ring_cons
= ACCESS_ONCE(ring
->cons
);
731 real_size
= get_real_size(skb
, shinfo
, dev
, &lso_header_size
,
732 &inline_ok
, &fragptr
);
733 if (unlikely(!real_size
))
736 /* Align descriptor to TXBB size */
737 desc_size
= ALIGN(real_size
, TXBB_SIZE
);
738 nr_txbb
= desc_size
/ TXBB_SIZE
;
739 if (unlikely(nr_txbb
> MAX_DESC_TXBBS
)) {
740 if (netif_msg_tx_err(priv
))
741 en_warn(priv
, "Oversized header or SG list\n");
745 if (vlan_tx_tag_present(skb
))
746 vlan_tag
= vlan_tx_tag_get(skb
);
749 netdev_txq_bql_enqueue_prefetchw(ring
->tx_queue
);
751 /* Track current inflight packets for performance analysis */
752 AVG_PERF_COUNTER(priv
->pstats
.inflight_avg
,
753 (u32
)(ring
->prod
- ring_cons
- 1));
755 /* Packet is good - grab an index and transmit it */
756 index
= ring
->prod
& ring
->size_mask
;
757 bf_index
= ring
->prod
;
759 /* See if we have enough space for whole descriptor TXBB for setting
760 * SW ownership on next descriptor; if not, use a bounce buffer. */
761 if (likely(index
+ nr_txbb
<= ring
->size
))
762 tx_desc
= ring
->buf
+ index
* TXBB_SIZE
;
764 tx_desc
= (struct mlx4_en_tx_desc
*) ring
->bounce_buf
;
768 /* Save skb in tx_info ring */
769 tx_info
= &ring
->tx_info
[index
];
771 tx_info
->nr_txbb
= nr_txbb
;
773 data
= &tx_desc
->data
;
775 data
= ((void *)&tx_desc
->lso
+ ALIGN(lso_header_size
+ 4,
778 /* valid only for none inline segments */
779 tx_info
->data_offset
= (void *)data
- (void *)tx_desc
;
781 tx_info
->inl
= inline_ok
;
783 tx_info
->linear
= (lso_header_size
< skb_headlen(skb
) &&
786 tx_info
->nr_maps
= shinfo
->nr_frags
+ tx_info
->linear
;
787 data
+= tx_info
->nr_maps
- 1;
793 /* Map fragments if any */
794 for (i_frag
= shinfo
->nr_frags
- 1; i_frag
>= 0; i_frag
--) {
795 const struct skb_frag_struct
*frag
;
797 frag
= &shinfo
->frags
[i_frag
];
798 byte_count
= skb_frag_size(frag
);
799 dma
= skb_frag_dma_map(ddev
, frag
,
802 if (dma_mapping_error(ddev
, dma
))
805 data
->addr
= cpu_to_be64(dma
);
806 data
->lkey
= ring
->mr_key
;
808 data
->byte_count
= cpu_to_be32(byte_count
);
812 /* Map linear part if needed */
813 if (tx_info
->linear
) {
814 byte_count
= skb_headlen(skb
) - lso_header_size
;
816 dma
= dma_map_single(ddev
, skb
->data
+
817 lso_header_size
, byte_count
,
819 if (dma_mapping_error(ddev
, dma
))
822 data
->addr
= cpu_to_be64(dma
);
823 data
->lkey
= ring
->mr_key
;
825 data
->byte_count
= cpu_to_be32(byte_count
);
827 /* tx completion can avoid cache line miss for common cases */
828 tx_info
->map0_dma
= dma
;
829 tx_info
->map0_byte_count
= byte_count
;
833 * For timestamping add flag to skb_shinfo and
834 * set flag for further reference
836 tx_info
->ts_requested
= 0;
837 if (unlikely(ring
->hwtstamp_tx_type
== HWTSTAMP_TX_ON
&&
838 shinfo
->tx_flags
& SKBTX_HW_TSTAMP
)) {
839 shinfo
->tx_flags
|= SKBTX_IN_PROGRESS
;
840 tx_info
->ts_requested
= 1;
843 /* Prepare ctrl segement apart opcode+ownership, which depends on
844 * whether LSO is used */
845 tx_desc
->ctrl
.srcrb_flags
= priv
->ctrl_flags
;
846 if (likely(skb
->ip_summed
== CHECKSUM_PARTIAL
)) {
847 if (!skb
->encapsulation
)
848 tx_desc
->ctrl
.srcrb_flags
|= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM
|
849 MLX4_WQE_CTRL_TCP_UDP_CSUM
);
851 tx_desc
->ctrl
.srcrb_flags
|= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM
);
855 if (priv
->flags
& MLX4_EN_FLAG_ENABLE_HW_LOOPBACK
) {
858 /* Copy dst mac address to wqe. This allows loopback in eSwitch,
859 * so that VFs and PF can communicate with each other
861 ethh
= (struct ethhdr
*)skb
->data
;
862 tx_desc
->ctrl
.srcrb_flags16
[0] = get_unaligned((__be16
*)ethh
->h_dest
);
863 tx_desc
->ctrl
.imm
= get_unaligned((__be32
*)(ethh
->h_dest
+ 2));
866 /* Handle LSO (TSO) packets */
867 if (lso_header_size
) {
870 /* Mark opcode as LSO */
871 op_own
= cpu_to_be32(MLX4_OPCODE_LSO
| (1 << 6)) |
872 ((ring
->prod
& ring
->size
) ?
873 cpu_to_be32(MLX4_EN_BIT_DESC_OWN
) : 0);
875 /* Fill in the LSO prefix */
876 tx_desc
->lso
.mss_hdr_size
= cpu_to_be32(
877 shinfo
->gso_size
<< 16 | lso_header_size
);
880 * note that we already verified that it is linear */
881 memcpy(tx_desc
->lso
.header
, skb
->data
, lso_header_size
);
885 i
= ((skb
->len
- lso_header_size
) / shinfo
->gso_size
) +
886 !!((skb
->len
- lso_header_size
) % shinfo
->gso_size
);
887 tx_info
->nr_bytes
= skb
->len
+ (i
- 1) * lso_header_size
;
890 /* Normal (Non LSO) packet */
891 op_own
= cpu_to_be32(MLX4_OPCODE_SEND
) |
892 ((ring
->prod
& ring
->size
) ?
893 cpu_to_be32(MLX4_EN_BIT_DESC_OWN
) : 0);
894 tx_info
->nr_bytes
= max_t(unsigned int, skb
->len
, ETH_ZLEN
);
897 ring
->bytes
+= tx_info
->nr_bytes
;
898 netdev_tx_sent_queue(ring
->tx_queue
, tx_info
->nr_bytes
);
899 AVG_PERF_COUNTER(priv
->pstats
.tx_pktsz_avg
, skb
->len
);
902 build_inline_wqe(tx_desc
, skb
, shinfo
, real_size
, &vlan_tag
,
905 if (skb
->encapsulation
) {
906 struct iphdr
*ipv4
= (struct iphdr
*)skb_inner_network_header(skb
);
907 if (ipv4
->protocol
== IPPROTO_TCP
|| ipv4
->protocol
== IPPROTO_UDP
)
908 op_own
|= cpu_to_be32(MLX4_WQE_CTRL_IIP
| MLX4_WQE_CTRL_ILP
);
910 op_own
|= cpu_to_be32(MLX4_WQE_CTRL_IIP
);
913 ring
->prod
+= nr_txbb
;
915 /* If we used a bounce buffer then copy descriptor back into place */
916 if (unlikely(bounce
))
917 tx_desc
= mlx4_en_bounce_to_desc(priv
, ring
, index
, desc_size
);
919 skb_tx_timestamp(skb
);
921 /* Check available TXBBs And 2K spare for prefetch */
922 stop_queue
= (int)(ring
->prod
- ring_cons
) >
923 ring
->size
- HEADROOM
- MAX_DESC_TXBBS
;
924 if (unlikely(stop_queue
)) {
925 netif_tx_stop_queue(ring
->tx_queue
);
926 ring
->queue_stopped
++;
928 send_doorbell
= !skb
->xmit_more
|| netif_xmit_stopped(ring
->tx_queue
);
930 real_size
= (real_size
/ 16) & 0x3f;
932 if (ring
->bf_enabled
&& desc_size
<= MAX_BF
&& !bounce
&&
933 !vlan_tx_tag_present(skb
) && send_doorbell
) {
934 tx_desc
->ctrl
.bf_qpn
= ring
->doorbell_qpn
|
935 cpu_to_be32(real_size
);
937 op_own
|= htonl((bf_index
& 0xffff) << 8);
938 /* Ensure new descriptor hits memory
939 * before setting ownership of this descriptor to HW
942 tx_desc
->ctrl
.owner_opcode
= op_own
;
946 mlx4_bf_copy(ring
->bf
.reg
+ ring
->bf
.offset
, &tx_desc
->ctrl
,
951 ring
->bf
.offset
^= ring
->bf
.buf_size
;
953 tx_desc
->ctrl
.vlan_tag
= cpu_to_be16(vlan_tag
);
954 tx_desc
->ctrl
.ins_vlan
= MLX4_WQE_CTRL_INS_VLAN
*
955 !!vlan_tx_tag_present(skb
);
956 tx_desc
->ctrl
.fence_size
= real_size
;
958 /* Ensure new descriptor hits memory
959 * before setting ownership of this descriptor to HW
962 tx_desc
->ctrl
.owner_opcode
= op_own
;
965 /* Since there is no iowrite*_native() that writes the
966 * value as is, without byteswapping - using the one
967 * the doesn't do byteswapping in the relevant arch
970 #if defined(__LITTLE_ENDIAN)
976 ring
->bf
.uar
->map
+ MLX4_SEND_DOORBELL
);
982 if (unlikely(stop_queue
)) {
983 /* If queue was emptied after the if (stop_queue) , and before
984 * the netif_tx_stop_queue() - need to wake the queue,
985 * or else it will remain stopped forever.
986 * Need a memory barrier to make sure ring->cons was not
987 * updated before queue was stopped.
991 ring_cons
= ACCESS_ONCE(ring
->cons
);
992 if (unlikely(((int)(ring
->prod
- ring_cons
)) <=
993 ring
->size
- HEADROOM
- MAX_DESC_TXBBS
)) {
994 netif_tx_wake_queue(ring
->tx_queue
);
1001 en_err(priv
, "DMA mapping error\n");
1003 while (++i_frag
< shinfo
->nr_frags
) {
1005 dma_unmap_page(ddev
, (dma_addr_t
) be64_to_cpu(data
->addr
),
1006 be32_to_cpu(data
->byte_count
),
1011 dev_kfree_skb_any(skb
);
1012 priv
->stats
.tx_dropped
++;
1013 return NETDEV_TX_OK
;