vhost-net: skip head management if no outstanding

[deliverable/linux.git] / drivers / vhost / net.c

diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 28ad7752e0f357ec9b89f099c0b762d30e70cf14..aa76ca72606a6cded4932aea5c8864f1ae2bba05 100644 (file)
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -83,6 +83,8 @@ struct vhost_net {
        /* Number of times zerocopy TX recently failed.
         * Protected by tx vq lock. */
        unsigned tx_zcopy_err;
+       /* Flush in progress. Protected by tx vq lock. */
+       bool tx_flush;
 };
 
 static void vhost_net_tx_packet(struct vhost_net *net)
@@ -101,7 +103,11 @@ static void vhost_net_tx_err(struct vhost_net *net)
 
 static bool vhost_net_tx_select_zcopy(struct vhost_net *net)
 {
-       return net->tx_packets / 64 >= net->tx_zcopy_err;
+       /* TX flush waits for outstanding DMAs to be done.
+        * Don't start new DMAs.
+        */
+       return !net->tx_flush &&
+               net->tx_packets / 64 >= net->tx_zcopy_err;
 }
 
 static bool vhost_sock_zcopy(struct socket *sock)
@@ -235,7 +241,7 @@ static void handle_tx(struct vhost_net *net)
        size_t hdr_size;
        struct socket *sock;
        struct vhost_ubuf_ref *uninitialized_var(ubufs);
-       bool zcopy;
+       bool zcopy, zcopy_used;
 
        /* TODO: check that we are running from vhost_worker? */
        sock = rcu_dereference_check(vq->private_data, 1);
@@ -313,8 +319,11 @@ static void handle_tx(struct vhost_net *net)
                               iov_length(vq->hdr, s), hdr_size);
                        break;
                }
+               zcopy_used = zcopy && (len >= VHOST_GOODCOPY_LEN ||
+                                      vq->upend_idx != vq->done_idx);
+
                /* use msg_control to pass vhost zerocopy ubuf info to skb */
-               if (zcopy) {
+               if (zcopy_used) {
                        vq->heads[vq->upend_idx].id = head;
                        if (!vhost_net_tx_select_zcopy(net) ||
                            len < VHOST_GOODCOPY_LEN) {
@@ -342,7 +351,7 @@ static void handle_tx(struct vhost_net *net)
                /* TODO: Check specific error and bomb out unless ENOBUFS? */
                err = sock->ops->sendmsg(NULL, sock, &msg, len);
                if (unlikely(err < 0)) {
-                       if (zcopy) {
+                       if (zcopy_used) {
                                if (ubufs)
                                        vhost_ubuf_put(ubufs);
                                vq->upend_idx = ((unsigned)vq->upend_idx - 1) %
@@ -356,7 +365,7 @@ static void handle_tx(struct vhost_net *net)
                if (err != len)
                        pr_debug("Truncated TX packet: "
                                 " len %d != %zd\n", err, len);
-               if (!zcopy)
+               if (!zcopy_used)
                        vhost_add_used_and_signal(&net->dev, vq, head, 0);
                else
                        vhost_zerocopy_signal_used(net, vq);
@@ -471,7 +480,8 @@ static void handle_rx(struct vhost_net *net)
                .hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE
        };
        size_t total_len = 0;
-       int err, headcount, mergeable;
+       int err, mergeable;
+       s16 headcount;
        size_t vhost_hlen, sock_hlen;
        size_t vhost_len, sock_len;
        /* TODO: check that we are running from vhost_worker? */
@@ -678,6 +688,17 @@ static void vhost_net_flush(struct vhost_net *n)
 {
        vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
        vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
+       if (n->dev.vqs[VHOST_NET_VQ_TX].ubufs) {
+               mutex_lock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex);
+               n->tx_flush = true;
+               mutex_unlock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex);
+               /* Wait for all lower device DMAs done. */
+               vhost_ubuf_put_and_wait(n->dev.vqs[VHOST_NET_VQ_TX].ubufs);
+               mutex_lock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex);
+               n->tx_flush = false;
+               kref_init(&n->dev.vqs[VHOST_NET_VQ_TX].ubufs->kref);
+               mutex_unlock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex);
+       }
 }
 
 static int vhost_net_release(struct inode *inode, struct file *f)
@@ -685,18 +706,10 @@ static int vhost_net_release(struct inode *inode, struct file *f)
        struct vhost_net *n = f->private_data;
        struct socket *tx_sock;
        struct socket *rx_sock;
-       int i;
 
        vhost_net_stop(n, &tx_sock, &rx_sock);
        vhost_net_flush(n);
        vhost_dev_stop(&n->dev);
-       for (i = 0; i < n->dev.nvqs; ++i) {
-               /* Wait for all lower device DMAs done. */
-               if (n->dev.vqs[i].ubufs)
-                       vhost_ubuf_put_and_wait(n->dev.vqs[i].ubufs);
-
-               vhost_zerocopy_signal_used(n, &n->dev.vqs[i]);
-       }
        vhost_dev_cleanup(&n->dev, false);
        if (tx_sock)
                fput(tx_sock->file);
@@ -822,6 +835,10 @@ static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
                r = vhost_init_used(vq);
                if (r)
                        goto err_vq;
+
+               n->tx_packets = 0;
+               n->tx_zcopy_err = 0;
+               n->tx_flush = false;
        }
 
        mutex_unlock(&vq->mutex);
@@ -938,8 +955,11 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
                return vhost_net_reset_owner(n);
        default:
                mutex_lock(&n->dev.mutex);
-               r = vhost_dev_ioctl(&n->dev, ioctl, arg);
-               vhost_net_flush(n);
+               r = vhost_dev_ioctl(&n->dev, ioctl, argp);
+               if (r == -ENOIOCTLCMD)
+                       r = vhost_vring_ioctl(&n->dev, ioctl, argp);
+               else
+                       vhost_net_flush(n);
                mutex_unlock(&n->dev.mutex);
                return r;
        }