aboutsummaryrefslogtreecommitdiff
path: root/drivers/virtio/virtio_ring.c
diff options
context:
space:
mode:
authorRusty Russell <rusty@rustcorp.com.au>2012-01-12 15:44:43 +1030
committerRusty Russell <rusty@rustcorp.com.au>2012-01-12 15:44:43 +1030
commitee7cd8981e15bcb365fc762afe3fc47b8242f630 (patch)
treedd680789368f78aae6fcfb521fe55d868eff3b5f /drivers/virtio/virtio_ring.c
parent3b720b8c865098c49c1570b6b5c7832bcfa6e6c2 (diff)
virtio: expose added descriptors immediately.
A virtio driver does virtqueue_add_buf() multiple times before finally calling virtqueue_kick(); previously we only exposed the added buffers in the virtqueue_kick() call. This means we don't need a memory barrier in virtqueue_add_buf(), but it reduces concurrency as the device (ie. host) can't see the buffers until the kick. In the unusual (but now possible) case where a driver does add_buf() and get_buf() without doing a kick, we do need to insert one before our counter wraps. Otherwise we could wrap num_added, and later on not realize that we have passed the marker where we should have kicked. Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Diffstat (limited to 'drivers/virtio/virtio_ring.c')
-rw-r--r--drivers/virtio/virtio_ring.c20
1 files changed, 14 insertions, 6 deletions
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 99dc9480f3f..36bb6a61372 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -251,9 +251,20 @@ add_head:
/* Put entry in available array (but don't update avail->idx until they
* do sync). */
- avail = ((vq->vring.avail->idx + vq->num_added++) & (vq->vring.num-1));
+ avail = (vq->vring.avail->idx & (vq->vring.num-1));
vq->vring.avail->ring[avail] = head;
+ /* Descriptors and available array need to be set before we expose the
+ * new available array entries. */
+ virtio_wmb(vq);
+ vq->vring.avail->idx++;
+ vq->num_added++;
+
+ /* This is very unlikely, but theoretically possible. Kick
+ * just in case. */
+ if (unlikely(vq->num_added == (1 << 16) - 1))
+ virtqueue_kick(_vq);
+
pr_debug("Added buffer head %i to %p\n", head, vq);
END_USE(vq);
@@ -283,13 +294,10 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
* new available array entries. */
virtio_wmb(vq);
- old = vq->vring.avail->idx;
- new = vq->vring.avail->idx = old + vq->num_added;
+ old = vq->vring.avail->idx - vq->num_added;
+ new = vq->vring.avail->idx;
vq->num_added = 0;
- /* Need to update avail index before checking if we should notify */
- virtio_mb(vq);
-
if (vq->event) {
needs_kick = vring_need_event(vring_avail_event(&vq->vring),
new, old);