2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
8 * Copyright (c) 2002-2005, K A Fraser
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
44 #include <xen/events.h>
45 #include <xen/interface/memory.h>
47 #include <asm/xen/hypercall.h>
48 #include <asm/xen/page.h>
50 /* Provide an option to disable split event channels at load time as
51 * event channels are limited resource. Split event channels are
54 bool separate_tx_rx_irq
= 1;
55 module_param(separate_tx_rx_irq
, bool, 0644);
58 * This is the maximum slots a skb can have. If a guest sends a skb
59 * which exceeds this limit it is considered malicious.
61 #define FATAL_SKB_SLOTS_DEFAULT 20
62 static unsigned int fatal_skb_slots
= FATAL_SKB_SLOTS_DEFAULT
;
63 module_param(fatal_skb_slots
, uint
, 0444);
66 * To avoid confusion, we define XEN_NETBK_LEGACY_SLOTS_MAX indicating
67 * the maximum slots a valid packet can use. Now this value is defined
68 * to be XEN_NETIF_NR_SLOTS_MIN, which is supposed to be supported by
71 #define XEN_NETBK_LEGACY_SLOTS_MAX XEN_NETIF_NR_SLOTS_MIN
74 * If head != INVALID_PENDING_RING_IDX, it means this tx request is head of
75 * one or more merged tx requests, otherwise it is the continuation of
76 * previous tx request.
78 static inline int pending_tx_is_head(struct xenvif
*vif
, RING_IDX idx
)
80 return vif
->pending_tx_info
[idx
].head
!= INVALID_PENDING_RING_IDX
;
83 static void xenvif_idx_release(struct xenvif
*vif
, u16 pending_idx
,
86 static void make_tx_response(struct xenvif
*vif
,
87 struct xen_netif_tx_request
*txp
,
90 static inline int tx_work_todo(struct xenvif
*vif
);
91 static inline int rx_work_todo(struct xenvif
*vif
);
93 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
100 static inline unsigned long idx_to_pfn(struct xenvif
*vif
,
103 return page_to_pfn(vif
->mmap_pages
[idx
]);
106 static inline unsigned long idx_to_kaddr(struct xenvif
*vif
,
109 return (unsigned long)pfn_to_kaddr(idx_to_pfn(vif
, idx
));
113 * This is the amount of packet we copy rather than map, so that the
114 * guest can't fiddle with the contents of the headers while we do
115 * packet processing on them (netfilter, routing, etc).
117 #define PKT_PROT_LEN (ETH_HLEN + \
119 sizeof(struct iphdr) + MAX_IPOPTLEN + \
120 sizeof(struct tcphdr) + MAX_TCP_OPTION_SPACE)
122 static u16
frag_get_pending_idx(skb_frag_t
*frag
)
124 return (u16
)frag
->page_offset
;
127 static void frag_set_pending_idx(skb_frag_t
*frag
, u16 pending_idx
)
129 frag
->page_offset
= pending_idx
;
132 static inline pending_ring_idx_t
pending_index(unsigned i
)
134 return i
& (MAX_PENDING_REQS
-1);
137 static inline pending_ring_idx_t
nr_pending_reqs(struct xenvif
*vif
)
139 return MAX_PENDING_REQS
-
140 vif
->pending_prod
+ vif
->pending_cons
;
143 static int max_required_rx_slots(struct xenvif
*vif
)
145 int max
= DIV_ROUND_UP(vif
->dev
->mtu
, PAGE_SIZE
);
147 /* XXX FIXME: RX path dependent on MAX_SKB_FRAGS */
148 if (vif
->can_sg
|| vif
->gso
|| vif
->gso_prefix
)
149 max
+= MAX_SKB_FRAGS
+ 1; /* extra_info + frags */
154 int xenvif_rx_ring_full(struct xenvif
*vif
)
156 RING_IDX peek
= vif
->rx_req_cons_peek
;
157 RING_IDX needed
= max_required_rx_slots(vif
);
159 return ((vif
->rx
.sring
->req_prod
- peek
) < needed
) ||
160 ((vif
->rx
.rsp_prod_pvt
+ XEN_NETIF_RX_RING_SIZE
- peek
) < needed
);
163 int xenvif_must_stop_queue(struct xenvif
*vif
)
165 if (!xenvif_rx_ring_full(vif
))
168 vif
->rx
.sring
->req_event
= vif
->rx_req_cons_peek
+
169 max_required_rx_slots(vif
);
170 mb(); /* request notification /then/ check the queue */
172 return xenvif_rx_ring_full(vif
);
176 * Returns true if we should start a new receive buffer instead of
177 * adding 'size' bytes to a buffer which currently contains 'offset'
180 static bool start_new_rx_buffer(int offset
, unsigned long size
, int head
)
182 /* simple case: we have completely filled the current buffer. */
183 if (offset
== MAX_BUFFER_OFFSET
)
187 * complex case: start a fresh buffer if the current frag
188 * would overflow the current buffer but only if:
189 * (i) this frag would fit completely in the next buffer
190 * and (ii) there is already some data in the current buffer
191 * and (iii) this is not the head buffer.
194 * - (i) stops us splitting a frag into two copies
195 * unless the frag is too large for a single buffer.
196 * - (ii) stops us from leaving a buffer pointlessly empty.
197 * - (iii) stops us leaving the first buffer
198 * empty. Strictly speaking this is already covered
199 * by (ii) but is explicitly checked because
200 * netfront relies on the first buffer being
201 * non-empty and can crash otherwise.
203 * This means we will effectively linearise small
204 * frags but do not needlessly split large buffers
205 * into multiple copies tend to give large frags their
206 * own buffers as before.
208 if ((offset
+ size
> MAX_BUFFER_OFFSET
) &&
209 (size
<= MAX_BUFFER_OFFSET
) && offset
&& !head
)
215 struct xenvif_count_slot_state
{
216 unsigned long copy_off
;
220 unsigned int xenvif_count_frag_slots(struct xenvif
*vif
,
221 unsigned long offset
, unsigned long size
,
222 struct xenvif_count_slot_state
*state
)
226 offset
&= ~PAGE_MASK
;
231 bytes
= PAGE_SIZE
- offset
;
236 if (start_new_rx_buffer(state
->copy_off
, bytes
, state
->head
)) {
241 if (state
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
242 bytes
= MAX_BUFFER_OFFSET
- state
->copy_off
;
244 state
->copy_off
+= bytes
;
249 if (offset
== PAGE_SIZE
)
259 * Figure out how many ring slots we're going to need to send @skb to
260 * the guest. This function is essentially a dry run of
261 * xenvif_gop_frag_copy.
263 unsigned int xenvif_count_skb_slots(struct xenvif
*vif
, struct sk_buff
*skb
)
265 struct xenvif_count_slot_state state
;
273 /* Slot for the first (partial) page of data. */
276 /* Need a slot for the GSO prefix for GSO extra data? */
277 if (skb_shinfo(skb
)->gso_size
)
281 while (data
< skb_tail_pointer(skb
)) {
282 unsigned long offset
= offset_in_page(data
);
283 unsigned long size
= PAGE_SIZE
- offset
;
285 if (data
+ size
> skb_tail_pointer(skb
))
286 size
= skb_tail_pointer(skb
) - data
;
288 count
+= xenvif_count_frag_slots(vif
, offset
, size
, &state
);
293 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
294 unsigned long size
= skb_frag_size(&skb_shinfo(skb
)->frags
[i
]);
295 unsigned long offset
= skb_shinfo(skb
)->frags
[i
].page_offset
;
297 count
+= xenvif_count_frag_slots(vif
, offset
, size
, &state
);
302 struct netrx_pending_operations
{
303 unsigned copy_prod
, copy_cons
;
304 unsigned meta_prod
, meta_cons
;
305 struct gnttab_copy
*copy
;
306 struct xenvif_rx_meta
*meta
;
308 grant_ref_t copy_gref
;
311 static struct xenvif_rx_meta
*get_next_rx_buffer(struct xenvif
*vif
,
312 struct netrx_pending_operations
*npo
)
314 struct xenvif_rx_meta
*meta
;
315 struct xen_netif_rx_request
*req
;
317 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
319 meta
= npo
->meta
+ npo
->meta_prod
++;
325 npo
->copy_gref
= req
->gref
;
331 * Set up the grant operations for this fragment. If it's a flipping
332 * interface, we also set up the unmap request from here.
334 static void xenvif_gop_frag_copy(struct xenvif
*vif
, struct sk_buff
*skb
,
335 struct netrx_pending_operations
*npo
,
336 struct page
*page
, unsigned long size
,
337 unsigned long offset
, int *head
)
339 struct gnttab_copy
*copy_gop
;
340 struct xenvif_rx_meta
*meta
;
343 /* Data must not cross a page boundary. */
344 BUG_ON(size
+ offset
> PAGE_SIZE
<<compound_order(page
));
346 meta
= npo
->meta
+ npo
->meta_prod
- 1;
348 /* Skip unused frames from start of page */
349 page
+= offset
>> PAGE_SHIFT
;
350 offset
&= ~PAGE_MASK
;
353 BUG_ON(offset
>= PAGE_SIZE
);
354 BUG_ON(npo
->copy_off
> MAX_BUFFER_OFFSET
);
356 bytes
= PAGE_SIZE
- offset
;
361 if (start_new_rx_buffer(npo
->copy_off
, bytes
, *head
)) {
363 * Netfront requires there to be some data in the head
368 meta
= get_next_rx_buffer(vif
, npo
);
371 if (npo
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
372 bytes
= MAX_BUFFER_OFFSET
- npo
->copy_off
;
374 copy_gop
= npo
->copy
+ npo
->copy_prod
++;
375 copy_gop
->flags
= GNTCOPY_dest_gref
;
376 copy_gop
->len
= bytes
;
378 copy_gop
->source
.domid
= DOMID_SELF
;
379 copy_gop
->source
.u
.gmfn
= virt_to_mfn(page_address(page
));
380 copy_gop
->source
.offset
= offset
;
382 copy_gop
->dest
.domid
= vif
->domid
;
383 copy_gop
->dest
.offset
= npo
->copy_off
;
384 copy_gop
->dest
.u
.ref
= npo
->copy_gref
;
386 npo
->copy_off
+= bytes
;
393 if (offset
== PAGE_SIZE
&& size
) {
394 BUG_ON(!PageCompound(page
));
399 /* Leave a gap for the GSO descriptor. */
400 if (*head
&& skb_shinfo(skb
)->gso_size
&& !vif
->gso_prefix
)
403 *head
= 0; /* There must be something in this buffer now. */
409 * Prepare an SKB to be transmitted to the frontend.
411 * This function is responsible for allocating grant operations, meta
414 * It returns the number of meta structures consumed. The number of
415 * ring slots used is always equal to the number of meta slots used
416 * plus the number of GSO descriptors used. Currently, we use either
417 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
418 * frontend-side LRO).
420 static int xenvif_gop_skb(struct sk_buff
*skb
,
421 struct netrx_pending_operations
*npo
)
423 struct xenvif
*vif
= netdev_priv(skb
->dev
);
424 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
426 struct xen_netif_rx_request
*req
;
427 struct xenvif_rx_meta
*meta
;
432 old_meta_prod
= npo
->meta_prod
;
434 /* Set up a GSO prefix descriptor, if necessary */
435 if (skb_shinfo(skb
)->gso_size
&& vif
->gso_prefix
) {
436 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
437 meta
= npo
->meta
+ npo
->meta_prod
++;
438 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
443 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
444 meta
= npo
->meta
+ npo
->meta_prod
++;
446 if (!vif
->gso_prefix
)
447 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
454 npo
->copy_gref
= req
->gref
;
457 while (data
< skb_tail_pointer(skb
)) {
458 unsigned int offset
= offset_in_page(data
);
459 unsigned int len
= PAGE_SIZE
- offset
;
461 if (data
+ len
> skb_tail_pointer(skb
))
462 len
= skb_tail_pointer(skb
) - data
;
464 xenvif_gop_frag_copy(vif
, skb
, npo
,
465 virt_to_page(data
), len
, offset
, &head
);
469 for (i
= 0; i
< nr_frags
; i
++) {
470 xenvif_gop_frag_copy(vif
, skb
, npo
,
471 skb_frag_page(&skb_shinfo(skb
)->frags
[i
]),
472 skb_frag_size(&skb_shinfo(skb
)->frags
[i
]),
473 skb_shinfo(skb
)->frags
[i
].page_offset
,
477 return npo
->meta_prod
- old_meta_prod
;
481 * This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
482 * used to set up the operations on the top of
483 * netrx_pending_operations, which have since been done. Check that
484 * they didn't give any errors and advance over them.
486 static int xenvif_check_gop(struct xenvif
*vif
, int nr_meta_slots
,
487 struct netrx_pending_operations
*npo
)
489 struct gnttab_copy
*copy_op
;
490 int status
= XEN_NETIF_RSP_OKAY
;
493 for (i
= 0; i
< nr_meta_slots
; i
++) {
494 copy_op
= npo
->copy
+ npo
->copy_cons
++;
495 if (copy_op
->status
!= GNTST_okay
) {
497 "Bad status %d from copy to DOM%d.\n",
498 copy_op
->status
, vif
->domid
);
499 status
= XEN_NETIF_RSP_ERROR
;
506 static void xenvif_add_frag_responses(struct xenvif
*vif
, int status
,
507 struct xenvif_rx_meta
*meta
,
511 unsigned long offset
;
513 /* No fragments used */
514 if (nr_meta_slots
<= 1)
519 for (i
= 0; i
< nr_meta_slots
; i
++) {
521 if (i
== nr_meta_slots
- 1)
524 flags
= XEN_NETRXF_more_data
;
527 make_rx_response(vif
, meta
[i
].id
, status
, offset
,
528 meta
[i
].size
, flags
);
532 struct skb_cb_overlay
{
536 static void xenvif_kick_thread(struct xenvif
*vif
)
541 void xenvif_rx_action(struct xenvif
*vif
)
545 struct xen_netif_rx_response
*resp
;
546 struct sk_buff_head rxq
;
552 unsigned long offset
;
553 struct skb_cb_overlay
*sco
;
554 int need_to_notify
= 0;
556 struct netrx_pending_operations npo
= {
557 .copy
= vif
->grant_copy_op
,
561 skb_queue_head_init(&rxq
);
565 while ((skb
= skb_dequeue(&vif
->rx_queue
)) != NULL
) {
566 vif
= netdev_priv(skb
->dev
);
567 nr_frags
= skb_shinfo(skb
)->nr_frags
;
569 sco
= (struct skb_cb_overlay
*)skb
->cb
;
570 sco
->meta_slots_used
= xenvif_gop_skb(skb
, &npo
);
572 count
+= nr_frags
+ 1;
574 __skb_queue_tail(&rxq
, skb
);
576 /* Filled the batch queue? */
577 /* XXX FIXME: RX path dependent on MAX_SKB_FRAGS */
578 if (count
+ MAX_SKB_FRAGS
>= XEN_NETIF_RX_RING_SIZE
)
582 BUG_ON(npo
.meta_prod
> ARRAY_SIZE(vif
->meta
));
587 BUG_ON(npo
.copy_prod
> ARRAY_SIZE(vif
->grant_copy_op
));
588 gnttab_batch_copy(vif
->grant_copy_op
, npo
.copy_prod
);
590 while ((skb
= __skb_dequeue(&rxq
)) != NULL
) {
591 sco
= (struct skb_cb_overlay
*)skb
->cb
;
593 vif
= netdev_priv(skb
->dev
);
595 if (vif
->meta
[npo
.meta_cons
].gso_size
&& vif
->gso_prefix
) {
596 resp
= RING_GET_RESPONSE(&vif
->rx
,
597 vif
->rx
.rsp_prod_pvt
++);
599 resp
->flags
= XEN_NETRXF_gso_prefix
| XEN_NETRXF_more_data
;
601 resp
->offset
= vif
->meta
[npo
.meta_cons
].gso_size
;
602 resp
->id
= vif
->meta
[npo
.meta_cons
].id
;
603 resp
->status
= sco
->meta_slots_used
;
606 sco
->meta_slots_used
--;
610 vif
->dev
->stats
.tx_bytes
+= skb
->len
;
611 vif
->dev
->stats
.tx_packets
++;
613 status
= xenvif_check_gop(vif
, sco
->meta_slots_used
, &npo
);
615 if (sco
->meta_slots_used
== 1)
618 flags
= XEN_NETRXF_more_data
;
620 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) /* local packet? */
621 flags
|= XEN_NETRXF_csum_blank
| XEN_NETRXF_data_validated
;
622 else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
)
623 /* remote but checksummed. */
624 flags
|= XEN_NETRXF_data_validated
;
627 resp
= make_rx_response(vif
, vif
->meta
[npo
.meta_cons
].id
,
629 vif
->meta
[npo
.meta_cons
].size
,
632 if (vif
->meta
[npo
.meta_cons
].gso_size
&& !vif
->gso_prefix
) {
633 struct xen_netif_extra_info
*gso
=
634 (struct xen_netif_extra_info
*)
635 RING_GET_RESPONSE(&vif
->rx
,
636 vif
->rx
.rsp_prod_pvt
++);
638 resp
->flags
|= XEN_NETRXF_extra_info
;
640 gso
->u
.gso
.size
= vif
->meta
[npo
.meta_cons
].gso_size
;
641 gso
->u
.gso
.type
= XEN_NETIF_GSO_TYPE_TCPV4
;
643 gso
->u
.gso
.features
= 0;
645 gso
->type
= XEN_NETIF_EXTRA_TYPE_GSO
;
649 xenvif_add_frag_responses(vif
, status
,
650 vif
->meta
+ npo
.meta_cons
+ 1,
651 sco
->meta_slots_used
);
653 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->rx
, ret
);
658 xenvif_notify_tx_completion(vif
);
660 npo
.meta_cons
+= sco
->meta_slots_used
;
665 notify_remote_via_irq(vif
->rx_irq
);
667 /* More work to do? */
668 if (!skb_queue_empty(&vif
->rx_queue
))
669 xenvif_kick_thread(vif
);
672 void xenvif_queue_tx_skb(struct xenvif
*vif
, struct sk_buff
*skb
)
674 skb_queue_tail(&vif
->rx_queue
, skb
);
676 xenvif_kick_thread(vif
);
679 void xenvif_check_rx_xenvif(struct xenvif
*vif
)
683 RING_FINAL_CHECK_FOR_REQUESTS(&vif
->tx
, more_to_do
);
686 napi_schedule(&vif
->napi
);
689 static void tx_add_credit(struct xenvif
*vif
)
691 unsigned long max_burst
, max_credit
;
694 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
695 * Otherwise the interface can seize up due to insufficient credit.
697 max_burst
= RING_GET_REQUEST(&vif
->tx
, vif
->tx
.req_cons
)->size
;
698 max_burst
= min(max_burst
, 131072UL);
699 max_burst
= max(max_burst
, vif
->credit_bytes
);
701 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
702 max_credit
= vif
->remaining_credit
+ vif
->credit_bytes
;
703 if (max_credit
< vif
->remaining_credit
)
704 max_credit
= ULONG_MAX
; /* wrapped: clamp to ULONG_MAX */
706 vif
->remaining_credit
= min(max_credit
, max_burst
);
709 static void tx_credit_callback(unsigned long data
)
711 struct xenvif
*vif
= (struct xenvif
*)data
;
713 xenvif_check_rx_xenvif(vif
);
716 static void xenvif_tx_err(struct xenvif
*vif
,
717 struct xen_netif_tx_request
*txp
, RING_IDX end
)
719 RING_IDX cons
= vif
->tx
.req_cons
;
722 make_tx_response(vif
, txp
, XEN_NETIF_RSP_ERROR
);
725 txp
= RING_GET_REQUEST(&vif
->tx
, cons
++);
727 vif
->tx
.req_cons
= cons
;
730 static void xenvif_fatal_tx_err(struct xenvif
*vif
)
732 netdev_err(vif
->dev
, "fatal error; disabling device\n");
733 xenvif_carrier_off(vif
);
736 static int xenvif_count_requests(struct xenvif
*vif
,
737 struct xen_netif_tx_request
*first
,
738 struct xen_netif_tx_request
*txp
,
741 RING_IDX cons
= vif
->tx
.req_cons
;
746 if (!(first
->flags
& XEN_NETTXF_more_data
))
750 struct xen_netif_tx_request dropped_tx
= { 0 };
752 if (slots
>= work_to_do
) {
754 "Asked for %d slots but exceeds this limit\n",
756 xenvif_fatal_tx_err(vif
);
760 /* This guest is really using too many slots and
761 * considered malicious.
763 if (unlikely(slots
>= fatal_skb_slots
)) {
765 "Malicious frontend using %d slots, threshold %u\n",
766 slots
, fatal_skb_slots
);
767 xenvif_fatal_tx_err(vif
);
771 /* Xen network protocol had implicit dependency on
772 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
773 * the historical MAX_SKB_FRAGS value 18 to honor the
774 * same behavior as before. Any packet using more than
775 * 18 slots but less than fatal_skb_slots slots is
778 if (!drop_err
&& slots
>= XEN_NETBK_LEGACY_SLOTS_MAX
) {
781 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
782 slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
789 memcpy(txp
, RING_GET_REQUEST(&vif
->tx
, cons
+ slots
),
792 /* If the guest submitted a frame >= 64 KiB then
793 * first->size overflowed and following slots will
794 * appear to be larger than the frame.
796 * This cannot be fatal error as there are buggy
797 * frontends that do this.
799 * Consume all slots and drop the packet.
801 if (!drop_err
&& txp
->size
> first
->size
) {
804 "Invalid tx request, slot size %u > remaining size %u\n",
805 txp
->size
, first
->size
);
809 first
->size
-= txp
->size
;
812 if (unlikely((txp
->offset
+ txp
->size
) > PAGE_SIZE
)) {
813 netdev_err(vif
->dev
, "Cross page boundary, txp->offset: %x, size: %u\n",
814 txp
->offset
, txp
->size
);
815 xenvif_fatal_tx_err(vif
);
819 more_data
= txp
->flags
& XEN_NETTXF_more_data
;
827 xenvif_tx_err(vif
, first
, cons
+ slots
);
834 static struct page
*xenvif_alloc_page(struct xenvif
*vif
,
839 page
= alloc_page(GFP_ATOMIC
|__GFP_COLD
);
842 vif
->mmap_pages
[pending_idx
] = page
;
847 static struct gnttab_copy
*xenvif_get_requests(struct xenvif
*vif
,
849 struct xen_netif_tx_request
*txp
,
850 struct gnttab_copy
*gop
)
852 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
853 skb_frag_t
*frags
= shinfo
->frags
;
854 u16 pending_idx
= *((u16
*)skb
->data
);
858 pending_ring_idx_t index
, start_idx
= 0;
860 unsigned int nr_slots
;
861 struct pending_tx_info
*first
= NULL
;
863 /* At this point shinfo->nr_frags is in fact the number of
864 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
866 nr_slots
= shinfo
->nr_frags
;
868 /* Skip first skb fragment if it is on same page as header fragment. */
869 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
871 /* Coalesce tx requests, at this point the packet passed in
872 * should be <= 64K. Any packets larger than 64K have been
873 * handled in xenvif_count_requests().
875 for (shinfo
->nr_frags
= slot
= start
; slot
< nr_slots
;
876 shinfo
->nr_frags
++) {
877 struct pending_tx_info
*pending_tx_info
=
878 vif
->pending_tx_info
;
880 page
= alloc_page(GFP_ATOMIC
|__GFP_COLD
);
886 while (dst_offset
< PAGE_SIZE
&& slot
< nr_slots
) {
887 gop
->flags
= GNTCOPY_source_gref
;
889 gop
->source
.u
.ref
= txp
->gref
;
890 gop
->source
.domid
= vif
->domid
;
891 gop
->source
.offset
= txp
->offset
;
893 gop
->dest
.domid
= DOMID_SELF
;
895 gop
->dest
.offset
= dst_offset
;
896 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
898 if (dst_offset
+ txp
->size
> PAGE_SIZE
) {
899 /* This page can only merge a portion
900 * of tx request. Do not increment any
901 * pointer / counter here. The txp
902 * will be dealt with in future
903 * rounds, eventually hitting the
906 gop
->len
= PAGE_SIZE
- dst_offset
;
907 txp
->offset
+= gop
->len
;
908 txp
->size
-= gop
->len
;
909 dst_offset
+= gop
->len
; /* quit loop */
911 /* This tx request can be merged in the page */
912 gop
->len
= txp
->size
;
913 dst_offset
+= gop
->len
;
915 index
= pending_index(vif
->pending_cons
++);
917 pending_idx
= vif
->pending_ring
[index
];
919 memcpy(&pending_tx_info
[pending_idx
].req
, txp
,
922 /* Poison these fields, corresponding
923 * fields for head tx req will be set
924 * to correct values after the loop.
926 vif
->mmap_pages
[pending_idx
] = (void *)(~0UL);
927 pending_tx_info
[pending_idx
].head
=
928 INVALID_PENDING_RING_IDX
;
931 first
= &pending_tx_info
[pending_idx
];
933 head_idx
= pending_idx
;
943 first
->req
.offset
= 0;
944 first
->req
.size
= dst_offset
;
945 first
->head
= start_idx
;
946 vif
->mmap_pages
[head_idx
] = page
;
947 frag_set_pending_idx(&frags
[shinfo
->nr_frags
], head_idx
);
950 BUG_ON(shinfo
->nr_frags
> MAX_SKB_FRAGS
);
954 /* Unwind, freeing all pages and sending error responses. */
955 while (shinfo
->nr_frags
-- > start
) {
956 xenvif_idx_release(vif
,
957 frag_get_pending_idx(&frags
[shinfo
->nr_frags
]),
958 XEN_NETIF_RSP_ERROR
);
960 /* The head too, if necessary. */
962 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_ERROR
);
967 static int xenvif_tx_check_gop(struct xenvif
*vif
,
969 struct gnttab_copy
**gopp
)
971 struct gnttab_copy
*gop
= *gopp
;
972 u16 pending_idx
= *((u16
*)skb
->data
);
973 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
974 struct pending_tx_info
*tx_info
;
975 int nr_frags
= shinfo
->nr_frags
;
977 u16 peek
; /* peek into next tx request */
979 /* Check status of header. */
982 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_ERROR
);
984 /* Skip first skb fragment if it is on same page as header fragment. */
985 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
987 for (i
= start
; i
< nr_frags
; i
++) {
989 pending_ring_idx_t head
;
991 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[i
]);
992 tx_info
= &vif
->pending_tx_info
[pending_idx
];
993 head
= tx_info
->head
;
995 /* Check error status: if okay then remember grant handle. */
997 newerr
= (++gop
)->status
;
1000 peek
= vif
->pending_ring
[pending_index(++head
)];
1001 } while (!pending_tx_is_head(vif
, peek
));
1003 if (likely(!newerr
)) {
1004 /* Had a previous error? Invalidate this fragment. */
1006 xenvif_idx_release(vif
, pending_idx
,
1007 XEN_NETIF_RSP_OKAY
);
1011 /* Error on this fragment: respond to client with an error. */
1012 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_ERROR
);
1014 /* Not the first error? Preceding frags already invalidated. */
1018 /* First error: invalidate header and preceding fragments. */
1019 pending_idx
= *((u16
*)skb
->data
);
1020 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1021 for (j
= start
; j
< i
; j
++) {
1022 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[j
]);
1023 xenvif_idx_release(vif
, pending_idx
,
1024 XEN_NETIF_RSP_OKAY
);
1027 /* Remember the error: invalidate all subsequent fragments. */
1035 static void xenvif_fill_frags(struct xenvif
*vif
, struct sk_buff
*skb
)
1037 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1038 int nr_frags
= shinfo
->nr_frags
;
1041 for (i
= 0; i
< nr_frags
; i
++) {
1042 skb_frag_t
*frag
= shinfo
->frags
+ i
;
1043 struct xen_netif_tx_request
*txp
;
1047 pending_idx
= frag_get_pending_idx(frag
);
1049 txp
= &vif
->pending_tx_info
[pending_idx
].req
;
1050 page
= virt_to_page(idx_to_kaddr(vif
, pending_idx
));
1051 __skb_fill_page_desc(skb
, i
, page
, txp
->offset
, txp
->size
);
1052 skb
->len
+= txp
->size
;
1053 skb
->data_len
+= txp
->size
;
1054 skb
->truesize
+= txp
->size
;
1056 /* Take an extra reference to offset xenvif_idx_release */
1057 get_page(vif
->mmap_pages
[pending_idx
]);
1058 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1062 static int xenvif_get_extras(struct xenvif
*vif
,
1063 struct xen_netif_extra_info
*extras
,
1066 struct xen_netif_extra_info extra
;
1067 RING_IDX cons
= vif
->tx
.req_cons
;
1070 if (unlikely(work_to_do
-- <= 0)) {
1071 netdev_err(vif
->dev
, "Missing extra info\n");
1072 xenvif_fatal_tx_err(vif
);
1076 memcpy(&extra
, RING_GET_REQUEST(&vif
->tx
, cons
),
1078 if (unlikely(!extra
.type
||
1079 extra
.type
>= XEN_NETIF_EXTRA_TYPE_MAX
)) {
1080 vif
->tx
.req_cons
= ++cons
;
1081 netdev_err(vif
->dev
,
1082 "Invalid extra type: %d\n", extra
.type
);
1083 xenvif_fatal_tx_err(vif
);
1087 memcpy(&extras
[extra
.type
- 1], &extra
, sizeof(extra
));
1088 vif
->tx
.req_cons
= ++cons
;
1089 } while (extra
.flags
& XEN_NETIF_EXTRA_FLAG_MORE
);
1094 static int xenvif_set_skb_gso(struct xenvif
*vif
,
1095 struct sk_buff
*skb
,
1096 struct xen_netif_extra_info
*gso
)
1098 if (!gso
->u
.gso
.size
) {
1099 netdev_err(vif
->dev
, "GSO size must not be zero.\n");
1100 xenvif_fatal_tx_err(vif
);
1104 /* Currently only TCPv4 S.O. is supported. */
1105 if (gso
->u
.gso
.type
!= XEN_NETIF_GSO_TYPE_TCPV4
) {
1106 netdev_err(vif
->dev
, "Bad GSO type %d.\n", gso
->u
.gso
.type
);
1107 xenvif_fatal_tx_err(vif
);
1111 skb_shinfo(skb
)->gso_size
= gso
->u
.gso
.size
;
1112 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
1114 /* Header must be checked, and gso_segs computed. */
1115 skb_shinfo(skb
)->gso_type
|= SKB_GSO_DODGY
;
1116 skb_shinfo(skb
)->gso_segs
= 0;
1121 static int checksum_setup(struct xenvif
*vif
, struct sk_buff
*skb
)
1125 int recalculate_partial_csum
= 0;
1128 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1129 * peers can fail to set NETRXF_csum_blank when sending a GSO
1130 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1131 * recalculate the partial checksum.
1133 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
&& skb_is_gso(skb
)) {
1134 vif
->rx_gso_checksum_fixup
++;
1135 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1136 recalculate_partial_csum
= 1;
1139 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1140 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1143 if (skb
->protocol
!= htons(ETH_P_IP
))
1146 iph
= (void *)skb
->data
;
1147 switch (iph
->protocol
) {
1149 if (!skb_partial_csum_set(skb
, 4 * iph
->ihl
,
1150 offsetof(struct tcphdr
, check
)))
1153 if (recalculate_partial_csum
) {
1154 struct tcphdr
*tcph
= tcp_hdr(skb
);
1155 tcph
->check
= ~csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
1156 skb
->len
- iph
->ihl
*4,
1161 if (!skb_partial_csum_set(skb
, 4 * iph
->ihl
,
1162 offsetof(struct udphdr
, check
)))
1165 if (recalculate_partial_csum
) {
1166 struct udphdr
*udph
= udp_hdr(skb
);
1167 udph
->check
= ~csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
1168 skb
->len
- iph
->ihl
*4,
1173 if (net_ratelimit())
1174 netdev_err(vif
->dev
,
1175 "Attempting to checksum a non-TCP/UDP packet, dropping a protocol %d packet\n",
1186 static bool tx_credit_exceeded(struct xenvif
*vif
, unsigned size
)
1188 unsigned long now
= jiffies
;
1189 unsigned long next_credit
=
1190 vif
->credit_timeout
.expires
+
1191 msecs_to_jiffies(vif
->credit_usec
/ 1000);
1193 /* Timer could already be pending in rare cases. */
1194 if (timer_pending(&vif
->credit_timeout
))
1197 /* Passed the point where we can replenish credit? */
1198 if (time_after_eq(now
, next_credit
)) {
1199 vif
->credit_timeout
.expires
= now
;
1203 /* Still too big to send right now? Set a callback. */
1204 if (size
> vif
->remaining_credit
) {
1205 vif
->credit_timeout
.data
=
1207 vif
->credit_timeout
.function
=
1209 mod_timer(&vif
->credit_timeout
,
1218 static unsigned xenvif_tx_build_gops(struct xenvif
*vif
)
1220 struct gnttab_copy
*gop
= vif
->tx_copy_ops
, *request_gop
;
1221 struct sk_buff
*skb
;
1224 while ((nr_pending_reqs(vif
) + XEN_NETBK_LEGACY_SLOTS_MAX
1225 < MAX_PENDING_REQS
)) {
1226 struct xen_netif_tx_request txreq
;
1227 struct xen_netif_tx_request txfrags
[XEN_NETBK_LEGACY_SLOTS_MAX
];
1229 struct xen_netif_extra_info extras
[XEN_NETIF_EXTRA_TYPE_MAX
-1];
1233 unsigned int data_len
;
1234 pending_ring_idx_t index
;
1236 if (vif
->tx
.sring
->req_prod
- vif
->tx
.req_cons
>
1237 XEN_NETIF_TX_RING_SIZE
) {
1238 netdev_err(vif
->dev
,
1239 "Impossible number of requests. "
1240 "req_prod %d, req_cons %d, size %ld\n",
1241 vif
->tx
.sring
->req_prod
, vif
->tx
.req_cons
,
1242 XEN_NETIF_TX_RING_SIZE
);
1243 xenvif_fatal_tx_err(vif
);
1247 RING_FINAL_CHECK_FOR_REQUESTS(&vif
->tx
, work_to_do
);
1251 idx
= vif
->tx
.req_cons
;
1252 rmb(); /* Ensure that we see the request before we copy it. */
1253 memcpy(&txreq
, RING_GET_REQUEST(&vif
->tx
, idx
), sizeof(txreq
));
1255 /* Credit-based scheduling. */
1256 if (txreq
.size
> vif
->remaining_credit
&&
1257 tx_credit_exceeded(vif
, txreq
.size
))
1260 vif
->remaining_credit
-= txreq
.size
;
1263 vif
->tx
.req_cons
= ++idx
;
1265 memset(extras
, 0, sizeof(extras
));
1266 if (txreq
.flags
& XEN_NETTXF_extra_info
) {
1267 work_to_do
= xenvif_get_extras(vif
, extras
,
1269 idx
= vif
->tx
.req_cons
;
1270 if (unlikely(work_to_do
< 0))
1274 ret
= xenvif_count_requests(vif
, &txreq
, txfrags
, work_to_do
);
1275 if (unlikely(ret
< 0))
1280 if (unlikely(txreq
.size
< ETH_HLEN
)) {
1281 netdev_dbg(vif
->dev
,
1282 "Bad packet size: %d\n", txreq
.size
);
1283 xenvif_tx_err(vif
, &txreq
, idx
);
1287 /* No crossing a page as the payload mustn't fragment. */
1288 if (unlikely((txreq
.offset
+ txreq
.size
) > PAGE_SIZE
)) {
1289 netdev_err(vif
->dev
,
1290 "txreq.offset: %x, size: %u, end: %lu\n",
1291 txreq
.offset
, txreq
.size
,
1292 (txreq
.offset
&~PAGE_MASK
) + txreq
.size
);
1293 xenvif_fatal_tx_err(vif
);
1297 index
= pending_index(vif
->pending_cons
);
1298 pending_idx
= vif
->pending_ring
[index
];
1300 data_len
= (txreq
.size
> PKT_PROT_LEN
&&
1301 ret
< XEN_NETBK_LEGACY_SLOTS_MAX
) ?
1302 PKT_PROT_LEN
: txreq
.size
;
1304 skb
= alloc_skb(data_len
+ NET_SKB_PAD
+ NET_IP_ALIGN
,
1305 GFP_ATOMIC
| __GFP_NOWARN
);
1306 if (unlikely(skb
== NULL
)) {
1307 netdev_dbg(vif
->dev
,
1308 "Can't allocate a skb in start_xmit.\n");
1309 xenvif_tx_err(vif
, &txreq
, idx
);
1313 /* Packets passed to netif_rx() must have some headroom. */
1314 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
);
1316 if (extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1].type
) {
1317 struct xen_netif_extra_info
*gso
;
1318 gso
= &extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1];
1320 if (xenvif_set_skb_gso(vif
, skb
, gso
)) {
1321 /* Failure in xenvif_set_skb_gso is fatal. */
1327 /* XXX could copy straight to head */
1328 page
= xenvif_alloc_page(vif
, pending_idx
);
1331 xenvif_tx_err(vif
, &txreq
, idx
);
1335 gop
->source
.u
.ref
= txreq
.gref
;
1336 gop
->source
.domid
= vif
->domid
;
1337 gop
->source
.offset
= txreq
.offset
;
1339 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
1340 gop
->dest
.domid
= DOMID_SELF
;
1341 gop
->dest
.offset
= txreq
.offset
;
1343 gop
->len
= txreq
.size
;
1344 gop
->flags
= GNTCOPY_source_gref
;
1348 memcpy(&vif
->pending_tx_info
[pending_idx
].req
,
1349 &txreq
, sizeof(txreq
));
1350 vif
->pending_tx_info
[pending_idx
].head
= index
;
1351 *((u16
*)skb
->data
) = pending_idx
;
1353 __skb_put(skb
, data_len
);
1355 skb_shinfo(skb
)->nr_frags
= ret
;
1356 if (data_len
< txreq
.size
) {
1357 skb_shinfo(skb
)->nr_frags
++;
1358 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1361 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1362 INVALID_PENDING_IDX
);
1365 vif
->pending_cons
++;
1367 request_gop
= xenvif_get_requests(vif
, skb
, txfrags
, gop
);
1368 if (request_gop
== NULL
) {
1370 xenvif_tx_err(vif
, &txreq
, idx
);
1375 __skb_queue_tail(&vif
->tx_queue
, skb
);
1377 vif
->tx
.req_cons
= idx
;
1379 if ((gop
-vif
->tx_copy_ops
) >= ARRAY_SIZE(vif
->tx_copy_ops
))
1383 return gop
- vif
->tx_copy_ops
;
1387 static int xenvif_tx_submit(struct xenvif
*vif
, int budget
)
1389 struct gnttab_copy
*gop
= vif
->tx_copy_ops
;
1390 struct sk_buff
*skb
;
1393 while (work_done
< budget
&&
1394 (skb
= __skb_dequeue(&vif
->tx_queue
)) != NULL
) {
1395 struct xen_netif_tx_request
*txp
;
1399 pending_idx
= *((u16
*)skb
->data
);
1400 txp
= &vif
->pending_tx_info
[pending_idx
].req
;
1402 /* Check the remap error code. */
1403 if (unlikely(xenvif_tx_check_gop(vif
, skb
, &gop
))) {
1404 netdev_dbg(vif
->dev
, "netback grant failed.\n");
1405 skb_shinfo(skb
)->nr_frags
= 0;
1410 data_len
= skb
->len
;
1412 (void *)(idx_to_kaddr(vif
, pending_idx
)|txp
->offset
),
1414 if (data_len
< txp
->size
) {
1415 /* Append the packet payload as a fragment. */
1416 txp
->offset
+= data_len
;
1417 txp
->size
-= data_len
;
1419 /* Schedule a response immediately. */
1420 xenvif_idx_release(vif
, pending_idx
,
1421 XEN_NETIF_RSP_OKAY
);
1424 if (txp
->flags
& XEN_NETTXF_csum_blank
)
1425 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1426 else if (txp
->flags
& XEN_NETTXF_data_validated
)
1427 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1429 xenvif_fill_frags(vif
, skb
);
1432 * If the initial fragment was < PKT_PROT_LEN then
1433 * pull through some bytes from the other fragments to
1434 * increase the linear region to PKT_PROT_LEN bytes.
1436 if (skb_headlen(skb
) < PKT_PROT_LEN
&& skb_is_nonlinear(skb
)) {
1437 int target
= min_t(int, skb
->len
, PKT_PROT_LEN
);
1438 __pskb_pull_tail(skb
, target
- skb_headlen(skb
));
1441 skb
->dev
= vif
->dev
;
1442 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1443 skb_reset_network_header(skb
);
1445 if (checksum_setup(vif
, skb
)) {
1446 netdev_dbg(vif
->dev
,
1447 "Can't setup checksum in net_tx_action\n");
1452 skb_probe_transport_header(skb
, 0);
1454 vif
->dev
->stats
.rx_bytes
+= skb
->len
;
1455 vif
->dev
->stats
.rx_packets
++;
1459 netif_receive_skb(skb
);
1465 /* Called after netfront has transmitted */
1466 int xenvif_tx_action(struct xenvif
*vif
, int budget
)
1471 if (unlikely(!tx_work_todo(vif
)))
1474 nr_gops
= xenvif_tx_build_gops(vif
);
1479 gnttab_batch_copy(vif
->tx_copy_ops
, nr_gops
);
1481 work_done
= xenvif_tx_submit(vif
, nr_gops
);
1486 static void xenvif_idx_release(struct xenvif
*vif
, u16 pending_idx
,
1489 struct pending_tx_info
*pending_tx_info
;
1490 pending_ring_idx_t head
;
1491 u16 peek
; /* peek into next tx request */
1493 BUG_ON(vif
->mmap_pages
[pending_idx
] == (void *)(~0UL));
1495 /* Already complete? */
1496 if (vif
->mmap_pages
[pending_idx
] == NULL
)
1499 pending_tx_info
= &vif
->pending_tx_info
[pending_idx
];
1501 head
= pending_tx_info
->head
;
1503 BUG_ON(!pending_tx_is_head(vif
, head
));
1504 BUG_ON(vif
->pending_ring
[pending_index(head
)] != pending_idx
);
1507 pending_ring_idx_t index
;
1508 pending_ring_idx_t idx
= pending_index(head
);
1509 u16 info_idx
= vif
->pending_ring
[idx
];
1511 pending_tx_info
= &vif
->pending_tx_info
[info_idx
];
1512 make_tx_response(vif
, &pending_tx_info
->req
, status
);
1514 /* Setting any number other than
1515 * INVALID_PENDING_RING_IDX indicates this slot is
1516 * starting a new packet / ending a previous packet.
1518 pending_tx_info
->head
= 0;
1520 index
= pending_index(vif
->pending_prod
++);
1521 vif
->pending_ring
[index
] = vif
->pending_ring
[info_idx
];
1523 peek
= vif
->pending_ring
[pending_index(++head
)];
1525 } while (!pending_tx_is_head(vif
, peek
));
1527 put_page(vif
->mmap_pages
[pending_idx
]);
1528 vif
->mmap_pages
[pending_idx
] = NULL
;
1532 static void make_tx_response(struct xenvif
*vif
,
1533 struct xen_netif_tx_request
*txp
,
1536 RING_IDX i
= vif
->tx
.rsp_prod_pvt
;
1537 struct xen_netif_tx_response
*resp
;
1540 resp
= RING_GET_RESPONSE(&vif
->tx
, i
);
1544 if (txp
->flags
& XEN_NETTXF_extra_info
)
1545 RING_GET_RESPONSE(&vif
->tx
, ++i
)->status
= XEN_NETIF_RSP_NULL
;
1547 vif
->tx
.rsp_prod_pvt
= ++i
;
1548 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->tx
, notify
);
1550 notify_remote_via_irq(vif
->tx_irq
);
1553 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
1560 RING_IDX i
= vif
->rx
.rsp_prod_pvt
;
1561 struct xen_netif_rx_response
*resp
;
1563 resp
= RING_GET_RESPONSE(&vif
->rx
, i
);
1564 resp
->offset
= offset
;
1565 resp
->flags
= flags
;
1567 resp
->status
= (s16
)size
;
1569 resp
->status
= (s16
)st
;
1571 vif
->rx
.rsp_prod_pvt
= ++i
;
1576 static inline int rx_work_todo(struct xenvif
*vif
)
1578 return !skb_queue_empty(&vif
->rx_queue
);
1581 static inline int tx_work_todo(struct xenvif
*vif
)
1584 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&vif
->tx
)) &&
1585 (nr_pending_reqs(vif
) + XEN_NETBK_LEGACY_SLOTS_MAX
1586 < MAX_PENDING_REQS
))
1592 void xenvif_unmap_frontend_rings(struct xenvif
*vif
)
1595 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1598 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1602 int xenvif_map_frontend_rings(struct xenvif
*vif
,
1603 grant_ref_t tx_ring_ref
,
1604 grant_ref_t rx_ring_ref
)
1607 struct xen_netif_tx_sring
*txs
;
1608 struct xen_netif_rx_sring
*rxs
;
1612 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1613 tx_ring_ref
, &addr
);
1617 txs
= (struct xen_netif_tx_sring
*)addr
;
1618 BACK_RING_INIT(&vif
->tx
, txs
, PAGE_SIZE
);
1620 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1621 rx_ring_ref
, &addr
);
1625 rxs
= (struct xen_netif_rx_sring
*)addr
;
1626 BACK_RING_INIT(&vif
->rx
, rxs
, PAGE_SIZE
);
1628 vif
->rx_req_cons_peek
= 0;
1633 xenvif_unmap_frontend_rings(vif
);
1637 int xenvif_kthread(void *data
)
1639 struct xenvif
*vif
= data
;
1641 while (!kthread_should_stop()) {
1642 wait_event_interruptible(vif
->wq
,
1643 rx_work_todo(vif
) ||
1644 kthread_should_stop());
1645 if (kthread_should_stop())
1648 if (rx_work_todo(vif
))
1649 xenvif_rx_action(vif
);
1657 static int __init
netback_init(void)
1664 if (fatal_skb_slots
< XEN_NETBK_LEGACY_SLOTS_MAX
) {
1665 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
1666 fatal_skb_slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
1667 fatal_skb_slots
= XEN_NETBK_LEGACY_SLOTS_MAX
;
1670 rc
= xenvif_xenbus_init();
1680 module_init(netback_init
);
1682 static void __exit
netback_fini(void)
1684 xenvif_xenbus_fini();
1686 module_exit(netback_fini
);
1688 MODULE_LICENSE("Dual BSD/GPL");
1689 MODULE_ALIAS("xen-backend:vif");