1 /* Copyright (C) 2009 Red Hat, Inc.
2 * Author: Michael S. Tsirkin <mst@redhat.com>
4 * This work is licensed under the terms of the GNU GPL, version 2.
6 * virtio-net server in host kernel.
9 #include <linux/compat.h>
10 #include <linux/eventfd.h>
11 #include <linux/vhost.h>
12 #include <linux/virtio_net.h>
13 #include <linux/mmu_context.h>
14 #include <linux/miscdevice.h>
15 #include <linux/module.h>
16 #include <linux/mutex.h>
17 #include <linux/workqueue.h>
18 #include <linux/rcupdate.h>
19 #include <linux/file.h>
20 #include <linux/slab.h>
22 #include <linux/net.h>
23 #include <linux/if_packet.h>
24 #include <linux/if_arp.h>
25 #include <linux/if_tun.h>
26 #include <linux/if_macvlan.h>
32 /* Max number of bytes transferred before requeueing the job.
33 * Using this limit prevents one virtqueue from starving others. */
34 #define VHOST_NET_WEIGHT 0x80000
42 enum vhost_net_poll_state
{
43 VHOST_NET_POLL_DISABLED
= 0,
44 VHOST_NET_POLL_STARTED
= 1,
45 VHOST_NET_POLL_STOPPED
= 2,
50 struct vhost_virtqueue vqs
[VHOST_NET_VQ_MAX
];
51 struct vhost_poll poll
[VHOST_NET_VQ_MAX
];
52 /* Tells us whether we are polling a socket for TX.
53 * We only do this when socket buffer fills up.
54 * Protected by tx vq lock. */
55 enum vhost_net_poll_state tx_poll_state
;
58 /* Pop first len bytes from iovec. Return number of segments used. */
59 static int move_iovec_hdr(struct iovec
*from
, struct iovec
*to
,
60 size_t len
, int iov_count
)
64 while (len
&& seg
< iov_count
) {
65 size
= min(from
->iov_len
, len
);
66 to
->iov_base
= from
->iov_base
;
68 from
->iov_len
-= size
;
69 from
->iov_base
+= size
;
78 /* Caller must have TX VQ lock */
79 static void tx_poll_stop(struct vhost_net
*net
)
81 if (likely(net
->tx_poll_state
!= VHOST_NET_POLL_STARTED
))
83 vhost_poll_stop(net
->poll
+ VHOST_NET_VQ_TX
);
84 net
->tx_poll_state
= VHOST_NET_POLL_STOPPED
;
87 /* Caller must have TX VQ lock */
88 static void tx_poll_start(struct vhost_net
*net
, struct socket
*sock
)
90 if (unlikely(net
->tx_poll_state
!= VHOST_NET_POLL_STOPPED
))
92 vhost_poll_start(net
->poll
+ VHOST_NET_VQ_TX
, sock
->file
);
93 net
->tx_poll_state
= VHOST_NET_POLL_STARTED
;
96 /* Expects to be always run from workqueue - which acts as
97 * read-size critical section for our kind of RCU. */
98 static void handle_tx(struct vhost_net
*net
)
100 struct vhost_virtqueue
*vq
= &net
->dev
.vqs
[VHOST_NET_VQ_TX
];
103 struct msghdr msg
= {
109 .msg_flags
= MSG_DONTWAIT
,
111 size_t len
, total_len
= 0;
114 struct socket
*sock
= rcu_dereference(vq
->private_data
);
118 wmem
= atomic_read(&sock
->sk
->sk_wmem_alloc
);
119 if (wmem
>= sock
->sk
->sk_sndbuf
) {
120 mutex_lock(&vq
->mutex
);
121 tx_poll_start(net
, sock
);
122 mutex_unlock(&vq
->mutex
);
127 mutex_lock(&vq
->mutex
);
128 vhost_disable_notify(vq
);
130 if (wmem
< sock
->sk
->sk_sndbuf
/ 2)
132 hdr_size
= vq
->hdr_size
;
135 head
= vhost_get_vq_desc(&net
->dev
, vq
, vq
->iov
,
139 /* On error, stop handling until the next kick. */
140 if (unlikely(head
< 0))
142 /* Nothing new? Wait for eventfd to tell us they refilled. */
143 if (head
== vq
->num
) {
144 wmem
= atomic_read(&sock
->sk
->sk_wmem_alloc
);
145 if (wmem
>= sock
->sk
->sk_sndbuf
* 3 / 4) {
146 tx_poll_start(net
, sock
);
147 set_bit(SOCK_ASYNC_NOSPACE
, &sock
->flags
);
150 if (unlikely(vhost_enable_notify(vq
))) {
151 vhost_disable_notify(vq
);
157 vq_err(vq
, "Unexpected descriptor format for TX: "
158 "out %d, int %d\n", out
, in
);
161 /* Skip header. TODO: support TSO. */
162 s
= move_iovec_hdr(vq
->iov
, vq
->hdr
, hdr_size
, out
);
163 msg
.msg_iovlen
= out
;
164 len
= iov_length(vq
->iov
, out
);
167 vq_err(vq
, "Unexpected header len for TX: "
168 "%zd expected %zd\n",
169 iov_length(vq
->hdr
, s
), hdr_size
);
172 /* TODO: Check specific error and bomb out unless ENOBUFS? */
173 err
= sock
->ops
->sendmsg(NULL
, sock
, &msg
, len
);
174 if (unlikely(err
< 0)) {
175 vhost_discard_vq_desc(vq
);
176 tx_poll_start(net
, sock
);
180 pr_debug("Truncated TX packet: "
181 " len %d != %zd\n", err
, len
);
182 vhost_add_used_and_signal(&net
->dev
, vq
, head
, 0);
184 if (unlikely(total_len
>= VHOST_NET_WEIGHT
)) {
185 vhost_poll_queue(&vq
->poll
);
190 mutex_unlock(&vq
->mutex
);
191 unuse_mm(net
->dev
.mm
);
194 /* Expects to be always run from workqueue - which acts as
195 * read-size critical section for our kind of RCU. */
196 static void handle_rx(struct vhost_net
*net
)
198 struct vhost_virtqueue
*vq
= &net
->dev
.vqs
[VHOST_NET_VQ_RX
];
199 unsigned out
, in
, log
, s
;
201 struct vhost_log
*vq_log
;
202 struct msghdr msg
= {
205 .msg_control
= NULL
, /* FIXME: get and handle RX aux data. */
208 .msg_flags
= MSG_DONTWAIT
,
211 struct virtio_net_hdr hdr
= {
213 .gso_type
= VIRTIO_NET_HDR_GSO_NONE
216 size_t len
, total_len
= 0;
219 struct socket
*sock
= rcu_dereference(vq
->private_data
);
220 if (!sock
|| skb_queue_empty(&sock
->sk
->sk_receive_queue
))
224 mutex_lock(&vq
->mutex
);
225 vhost_disable_notify(vq
);
226 hdr_size
= vq
->hdr_size
;
228 vq_log
= unlikely(vhost_has_feature(&net
->dev
, VHOST_F_LOG_ALL
)) ?
232 head
= vhost_get_vq_desc(&net
->dev
, vq
, vq
->iov
,
236 /* On error, stop handling until the next kick. */
237 if (unlikely(head
< 0))
239 /* OK, now we need to know about added descriptors. */
240 if (head
== vq
->num
) {
241 if (unlikely(vhost_enable_notify(vq
))) {
242 /* They have slipped one in as we were
243 * doing that: check again. */
244 vhost_disable_notify(vq
);
247 /* Nothing new? Wait for eventfd to tell us
251 /* We don't need to be notified again. */
253 vq_err(vq
, "Unexpected descriptor format for RX: "
258 /* Skip header. TODO: support TSO/mergeable rx buffers. */
259 s
= move_iovec_hdr(vq
->iov
, vq
->hdr
, hdr_size
, in
);
261 len
= iov_length(vq
->iov
, in
);
264 vq_err(vq
, "Unexpected header len for RX: "
265 "%zd expected %zd\n",
266 iov_length(vq
->hdr
, s
), hdr_size
);
269 err
= sock
->ops
->recvmsg(NULL
, sock
, &msg
,
270 len
, MSG_DONTWAIT
| MSG_TRUNC
);
271 /* TODO: Check specific error and bomb out unless EAGAIN? */
273 vhost_discard_vq_desc(vq
);
276 /* TODO: Should check and handle checksum. */
278 pr_debug("Discarded truncated rx packet: "
279 " len %d > %zd\n", err
, len
);
280 vhost_discard_vq_desc(vq
);
284 err
= memcpy_toiovec(vq
->hdr
, (unsigned char *)&hdr
, hdr_size
);
286 vq_err(vq
, "Unable to write vnet_hdr at addr %p: %d\n",
287 vq
->iov
->iov_base
, err
);
291 vhost_add_used_and_signal(&net
->dev
, vq
, head
, len
);
292 if (unlikely(vq_log
))
293 vhost_log_write(vq
, vq_log
, log
, len
);
295 if (unlikely(total_len
>= VHOST_NET_WEIGHT
)) {
296 vhost_poll_queue(&vq
->poll
);
301 mutex_unlock(&vq
->mutex
);
302 unuse_mm(net
->dev
.mm
);
305 static void handle_tx_kick(struct work_struct
*work
)
307 struct vhost_virtqueue
*vq
;
308 struct vhost_net
*net
;
309 vq
= container_of(work
, struct vhost_virtqueue
, poll
.work
);
310 net
= container_of(vq
->dev
, struct vhost_net
, dev
);
314 static void handle_rx_kick(struct work_struct
*work
)
316 struct vhost_virtqueue
*vq
;
317 struct vhost_net
*net
;
318 vq
= container_of(work
, struct vhost_virtqueue
, poll
.work
);
319 net
= container_of(vq
->dev
, struct vhost_net
, dev
);
323 static void handle_tx_net(struct work_struct
*work
)
325 struct vhost_net
*net
;
326 net
= container_of(work
, struct vhost_net
, poll
[VHOST_NET_VQ_TX
].work
);
330 static void handle_rx_net(struct work_struct
*work
)
332 struct vhost_net
*net
;
333 net
= container_of(work
, struct vhost_net
, poll
[VHOST_NET_VQ_RX
].work
);
337 static int vhost_net_open(struct inode
*inode
, struct file
*f
)
339 struct vhost_net
*n
= kmalloc(sizeof *n
, GFP_KERNEL
);
343 n
->vqs
[VHOST_NET_VQ_TX
].handle_kick
= handle_tx_kick
;
344 n
->vqs
[VHOST_NET_VQ_RX
].handle_kick
= handle_rx_kick
;
345 r
= vhost_dev_init(&n
->dev
, n
->vqs
, VHOST_NET_VQ_MAX
);
351 vhost_poll_init(n
->poll
+ VHOST_NET_VQ_TX
, handle_tx_net
, POLLOUT
);
352 vhost_poll_init(n
->poll
+ VHOST_NET_VQ_RX
, handle_rx_net
, POLLIN
);
353 n
->tx_poll_state
= VHOST_NET_POLL_DISABLED
;
360 static void vhost_net_disable_vq(struct vhost_net
*n
,
361 struct vhost_virtqueue
*vq
)
363 if (!vq
->private_data
)
365 if (vq
== n
->vqs
+ VHOST_NET_VQ_TX
) {
367 n
->tx_poll_state
= VHOST_NET_POLL_DISABLED
;
369 vhost_poll_stop(n
->poll
+ VHOST_NET_VQ_RX
);
372 static void vhost_net_enable_vq(struct vhost_net
*n
,
373 struct vhost_virtqueue
*vq
)
375 struct socket
*sock
= vq
->private_data
;
378 if (vq
== n
->vqs
+ VHOST_NET_VQ_TX
) {
379 n
->tx_poll_state
= VHOST_NET_POLL_STOPPED
;
380 tx_poll_start(n
, sock
);
382 vhost_poll_start(n
->poll
+ VHOST_NET_VQ_RX
, sock
->file
);
385 static struct socket
*vhost_net_stop_vq(struct vhost_net
*n
,
386 struct vhost_virtqueue
*vq
)
390 mutex_lock(&vq
->mutex
);
391 sock
= vq
->private_data
;
392 vhost_net_disable_vq(n
, vq
);
393 rcu_assign_pointer(vq
->private_data
, NULL
);
394 mutex_unlock(&vq
->mutex
);
398 static void vhost_net_stop(struct vhost_net
*n
, struct socket
**tx_sock
,
399 struct socket
**rx_sock
)
401 *tx_sock
= vhost_net_stop_vq(n
, n
->vqs
+ VHOST_NET_VQ_TX
);
402 *rx_sock
= vhost_net_stop_vq(n
, n
->vqs
+ VHOST_NET_VQ_RX
);
405 static void vhost_net_flush_vq(struct vhost_net
*n
, int index
)
407 vhost_poll_flush(n
->poll
+ index
);
408 vhost_poll_flush(&n
->dev
.vqs
[index
].poll
);
411 static void vhost_net_flush(struct vhost_net
*n
)
413 vhost_net_flush_vq(n
, VHOST_NET_VQ_TX
);
414 vhost_net_flush_vq(n
, VHOST_NET_VQ_RX
);
417 static int vhost_net_release(struct inode
*inode
, struct file
*f
)
419 struct vhost_net
*n
= f
->private_data
;
420 struct socket
*tx_sock
;
421 struct socket
*rx_sock
;
423 vhost_net_stop(n
, &tx_sock
, &rx_sock
);
425 vhost_dev_cleanup(&n
->dev
);
430 /* We do an extra flush before freeing memory,
431 * since jobs can re-queue themselves. */
437 static struct socket
*get_raw_socket(int fd
)
440 struct sockaddr_ll sa
;
441 char buf
[MAX_ADDR_LEN
];
443 int uaddr_len
= sizeof uaddr
, r
;
444 struct socket
*sock
= sockfd_lookup(fd
, &r
);
446 return ERR_PTR(-ENOTSOCK
);
448 /* Parameter checking */
449 if (sock
->sk
->sk_type
!= SOCK_RAW
) {
450 r
= -ESOCKTNOSUPPORT
;
454 r
= sock
->ops
->getname(sock
, (struct sockaddr
*)&uaddr
.sa
,
459 if (uaddr
.sa
.sll_family
!= AF_PACKET
) {
469 static struct socket
*get_tap_socket(int fd
)
471 struct file
*file
= fget(fd
);
474 return ERR_PTR(-EBADF
);
475 sock
= tun_get_socket(file
);
478 sock
= macvtap_get_socket(file
);
484 static struct socket
*get_socket(int fd
)
487 /* special case to disable backend */
490 sock
= get_raw_socket(fd
);
493 sock
= get_tap_socket(fd
);
496 return ERR_PTR(-ENOTSOCK
);
499 static long vhost_net_set_backend(struct vhost_net
*n
, unsigned index
, int fd
)
501 struct socket
*sock
, *oldsock
;
502 struct vhost_virtqueue
*vq
;
505 mutex_lock(&n
->dev
.mutex
);
506 r
= vhost_dev_check_owner(&n
->dev
);
510 if (index
>= VHOST_NET_VQ_MAX
) {
515 mutex_lock(&vq
->mutex
);
517 /* Verify that ring has been setup correctly. */
518 if (!vhost_vq_access_ok(vq
)) {
522 sock
= get_socket(fd
);
528 /* start polling new socket */
529 oldsock
= vq
->private_data
;
533 vhost_net_disable_vq(n
, vq
);
534 rcu_assign_pointer(vq
->private_data
, sock
);
535 vhost_net_enable_vq(n
, vq
);
537 mutex_unlock(&vq
->mutex
);
540 vhost_net_flush_vq(n
, index
);
544 mutex_unlock(&n
->dev
.mutex
);
548 mutex_unlock(&vq
->mutex
);
550 mutex_unlock(&n
->dev
.mutex
);
554 static long vhost_net_reset_owner(struct vhost_net
*n
)
556 struct socket
*tx_sock
= NULL
;
557 struct socket
*rx_sock
= NULL
;
559 mutex_lock(&n
->dev
.mutex
);
560 err
= vhost_dev_check_owner(&n
->dev
);
563 vhost_net_stop(n
, &tx_sock
, &rx_sock
);
565 err
= vhost_dev_reset_owner(&n
->dev
);
567 mutex_unlock(&n
->dev
.mutex
);
575 static int vhost_net_set_features(struct vhost_net
*n
, u64 features
)
577 size_t hdr_size
= features
& (1 << VHOST_NET_F_VIRTIO_NET_HDR
) ?
578 sizeof(struct virtio_net_hdr
) : 0;
580 mutex_lock(&n
->dev
.mutex
);
581 if ((features
& (1 << VHOST_F_LOG_ALL
)) &&
582 !vhost_log_access_ok(&n
->dev
)) {
583 mutex_unlock(&n
->dev
.mutex
);
586 n
->dev
.acked_features
= features
;
588 for (i
= 0; i
< VHOST_NET_VQ_MAX
; ++i
) {
589 mutex_lock(&n
->vqs
[i
].mutex
);
590 n
->vqs
[i
].hdr_size
= hdr_size
;
591 mutex_unlock(&n
->vqs
[i
].mutex
);
594 mutex_unlock(&n
->dev
.mutex
);
598 static long vhost_net_ioctl(struct file
*f
, unsigned int ioctl
,
601 struct vhost_net
*n
= f
->private_data
;
602 void __user
*argp
= (void __user
*)arg
;
603 u64 __user
*featurep
= argp
;
604 struct vhost_vring_file backend
;
608 case VHOST_NET_SET_BACKEND
:
609 if (copy_from_user(&backend
, argp
, sizeof backend
))
611 return vhost_net_set_backend(n
, backend
.index
, backend
.fd
);
612 case VHOST_GET_FEATURES
:
613 features
= VHOST_FEATURES
;
614 if (copy_to_user(featurep
, &features
, sizeof features
))
617 case VHOST_SET_FEATURES
:
618 if (copy_from_user(&features
, featurep
, sizeof features
))
620 if (features
& ~VHOST_FEATURES
)
622 return vhost_net_set_features(n
, features
);
623 case VHOST_RESET_OWNER
:
624 return vhost_net_reset_owner(n
);
626 mutex_lock(&n
->dev
.mutex
);
627 r
= vhost_dev_ioctl(&n
->dev
, ioctl
, arg
);
629 mutex_unlock(&n
->dev
.mutex
);
635 static long vhost_net_compat_ioctl(struct file
*f
, unsigned int ioctl
,
638 return vhost_net_ioctl(f
, ioctl
, (unsigned long)compat_ptr(arg
));
642 const static struct file_operations vhost_net_fops
= {
643 .owner
= THIS_MODULE
,
644 .release
= vhost_net_release
,
645 .unlocked_ioctl
= vhost_net_ioctl
,
647 .compat_ioctl
= vhost_net_compat_ioctl
,
649 .open
= vhost_net_open
,
652 static struct miscdevice vhost_net_misc
= {
658 static int vhost_net_init(void)
660 int r
= vhost_init();
663 r
= misc_register(&vhost_net_misc
);
673 module_init(vhost_net_init
);
675 static void vhost_net_exit(void)
677 misc_deregister(&vhost_net_misc
);
680 module_exit(vhost_net_exit
);
682 MODULE_VERSION("0.0.1");
683 MODULE_LICENSE("GPL v2");
684 MODULE_AUTHOR("Michael S. Tsirkin");
685 MODULE_DESCRIPTION("Host kernel accelerator for virtio net");