/* Number of times zerocopy TX recently failed.
* Protected by tx vq lock. */
unsigned tx_zcopy_err;
+ /* Flush in progress. Protected by tx vq lock. */
+ bool tx_flush;
};
static void vhost_net_tx_packet(struct vhost_net *net)
static bool vhost_net_tx_select_zcopy(struct vhost_net *net)
{
- return net->tx_packets / 64 >= net->tx_zcopy_err;
+ /* TX flush waits for outstanding DMAs to be done.
+ * Don't start new DMAs.
+ */
+ return !net->tx_flush &&
+ net->tx_packets / 64 >= net->tx_zcopy_err;
}
static bool vhost_sock_zcopy(struct socket *sock)
{
struct vhost_ubuf_ref *ubufs = ubuf->ctx;
struct vhost_virtqueue *vq = ubufs->vq;
-
- vhost_poll_queue(&vq->poll);
+ int cnt = atomic_read(&ubufs->kref.refcount);
+
+ /*
+ * Trigger polling thread if guest stopped submitting new buffers:
+ * in this case, the refcount after decrement will eventually reach 1
+ * so here it is 2.
+ * We also trigger polling periodically after each 16 packets
+ * (the value 16 here is more or less arbitrary, it's tuned to trigger
+ * less than 10% of times).
+ */
+ if (cnt <= 2 || !(cnt % 16))
+ vhost_poll_queue(&vq->poll);
/* set len to mark this desc buffers done DMA */
vq->heads[ubuf->desc].len = success ?
VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN;
size_t hdr_size;
struct socket *sock;
struct vhost_ubuf_ref *uninitialized_var(ubufs);
- bool zcopy;
+ bool zcopy, zcopy_used;
/* TODO: check that we are running from vhost_worker? */
sock = rcu_dereference_check(vq->private_data, 1);
iov_length(vq->hdr, s), hdr_size);
break;
}
+ zcopy_used = zcopy && (len >= VHOST_GOODCOPY_LEN ||
+ vq->upend_idx != vq->done_idx);
+
/* use msg_control to pass vhost zerocopy ubuf info to skb */
- if (zcopy) {
+ if (zcopy_used) {
vq->heads[vq->upend_idx].id = head;
if (!vhost_net_tx_select_zcopy(net) ||
len < VHOST_GOODCOPY_LEN) {
/* TODO: Check specific error and bomb out unless ENOBUFS? */
err = sock->ops->sendmsg(NULL, sock, &msg, len);
if (unlikely(err < 0)) {
- if (zcopy) {
+ if (zcopy_used) {
if (ubufs)
vhost_ubuf_put(ubufs);
vq->upend_idx = ((unsigned)vq->upend_idx - 1) %
if (err != len)
pr_debug("Truncated TX packet: "
" len %d != %zd\n", err, len);
- if (!zcopy)
+ if (!zcopy_used)
vhost_add_used_and_signal(&net->dev, vq, head, 0);
else
vhost_zerocopy_signal_used(net, vq);
.hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE
};
size_t total_len = 0;
- int err, headcount, mergeable;
+ int err, mergeable;
+ s16 headcount;
size_t vhost_hlen, sock_hlen;
size_t vhost_len, sock_len;
/* TODO: check that we are running from vhost_worker? */
{
vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
+ if (n->dev.vqs[VHOST_NET_VQ_TX].ubufs) {
+ mutex_lock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex);
+ n->tx_flush = true;
+ mutex_unlock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex);
+ /* Wait for all lower device DMAs done. */
+ vhost_ubuf_put_and_wait(n->dev.vqs[VHOST_NET_VQ_TX].ubufs);
+ mutex_lock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex);
+ n->tx_flush = false;
+ kref_init(&n->dev.vqs[VHOST_NET_VQ_TX].ubufs->kref);
+ mutex_unlock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex);
+ }
}
static int vhost_net_release(struct inode *inode, struct file *f)
struct vhost_net *n = f->private_data;
struct socket *tx_sock;
struct socket *rx_sock;
- int i;
vhost_net_stop(n, &tx_sock, &rx_sock);
vhost_net_flush(n);
vhost_dev_stop(&n->dev);
- for (i = 0; i < n->dev.nvqs; ++i) {
- /* Wait for all lower device DMAs done. */
- if (n->dev.vqs[i].ubufs)
- vhost_ubuf_put_and_wait(n->dev.vqs[i].ubufs);
-
- vhost_zerocopy_signal_used(n, &n->dev.vqs[i]);
- }
vhost_dev_cleanup(&n->dev, false);
if (tx_sock)
fput(tx_sock->file);
r = vhost_init_used(vq);
if (r)
goto err_vq;
+
+ n->tx_packets = 0;
+ n->tx_zcopy_err = 0;
+ n->tx_flush = false;
}
mutex_unlock(&vq->mutex);
return vhost_net_reset_owner(n);
default:
mutex_lock(&n->dev.mutex);
- r = vhost_dev_ioctl(&n->dev, ioctl, arg);
- vhost_net_flush(n);
+ r = vhost_dev_ioctl(&n->dev, ioctl, argp);
+ if (r == -ENOIOCTLCMD)
+ r = vhost_vring_ioctl(&n->dev, ioctl, argp);
+ else
+ vhost_net_flush(n);
mutex_unlock(&n->dev.mutex);
return r;
}