aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--drivers/lguest/lguest_device.c8
-rw-r--r--drivers/s390/kvm/kvm_virtio.c2
-rw-r--r--drivers/virtio/virtio_mmio.c4
-rw-r--r--drivers/virtio/virtio_pci.c4
-rw-r--r--drivers/virtio/virtio_ring.c34
-rw-r--r--include/linux/virtio_ring.h1
-rw-r--r--tools/virtio/linux/virtio.h1
-rw-r--r--tools/virtio/virtio_test.c3
8 files changed, 35 insertions, 22 deletions
diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
index 595d73197016..6a1d6447b864 100644
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -292,10 +292,12 @@ static struct virtqueue *lg_find_vq(struct virtio_device *vdev,
292 292
293 /* 293 /*
294 * OK, tell virtio_ring.c to set up a virtqueue now we know its size 294 * OK, tell virtio_ring.c to set up a virtqueue now we know its size
295 * and we've got a pointer to its pages. 295 * and we've got a pointer to its pages. Note that we set weak_barriers
296 * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
297 * barriers.
296 */ 298 */
297 vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, 299 vq = vring_new_virtqueue(lvq->config.num, LGUEST_VRING_ALIGN, vdev,
298 vdev, lvq->pages, lg_notify, callback, name); 300 true, lvq->pages, lg_notify, callback, name);
299 if (!vq) { 301 if (!vq) {
300 err = -ENOMEM; 302 err = -ENOMEM;
301 goto unmap; 303 goto unmap;
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
index 8af868bab20b..7bc1955337ea 100644
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -198,7 +198,7 @@ static struct virtqueue *kvm_find_vq(struct virtio_device *vdev,
198 goto out; 198 goto out;
199 199
200 vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN, 200 vq = vring_new_virtqueue(config->num, KVM_S390_VIRTIO_RING_ALIGN,
201 vdev, (void *) config->address, 201 vdev, true, (void *) config->address,
202 kvm_notify, callback, name); 202 kvm_notify, callback, name);
203 if (!vq) { 203 if (!vq) {
204 err = -ENOMEM; 204 err = -ENOMEM;
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index 0269717436af..01d6dc250d5c 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -310,8 +310,8 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index,
310 vm_dev->base + VIRTIO_MMIO_QUEUE_PFN); 310 vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
311 311
312 /* Create the vring */ 312 /* Create the vring */
313 vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, 313 vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
314 vdev, info->queue, vm_notify, callback, name); 314 true, info->queue, vm_notify, callback, name);
315 if (!vq) { 315 if (!vq) {
316 err = -ENOMEM; 316 err = -ENOMEM;
317 goto error_new_virtqueue; 317 goto error_new_virtqueue;
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
index baabb7937ec2..688b42d28dad 100644
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -414,8 +414,8 @@ static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
414 vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); 414 vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
415 415
416 /* create the vring */ 416 /* create the vring */
417 vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, 417 vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
418 vdev, info->queue, vp_notify, callback, name); 418 true, info->queue, vp_notify, callback, name);
419 if (!vq) { 419 if (!vq) {
420 err = -ENOMEM; 420 err = -ENOMEM;
421 goto out_activate_queue; 421 goto out_activate_queue;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index c7a2c208f6ea..50da92046092 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -28,17 +28,20 @@
28#ifdef CONFIG_SMP 28#ifdef CONFIG_SMP
29/* Where possible, use SMP barriers which are more lightweight than mandatory 29/* Where possible, use SMP barriers which are more lightweight than mandatory
30 * barriers, because mandatory barriers control MMIO effects on accesses 30 * barriers, because mandatory barriers control MMIO effects on accesses
31 * through relaxed memory I/O windows (which virtio does not use). */ 31 * through relaxed memory I/O windows (which virtio-pci does not use). */
32#define virtio_mb() smp_mb() 32#define virtio_mb(vq) \
33#define virtio_rmb() smp_rmb() 33 do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0)
34#define virtio_wmb() smp_wmb() 34#define virtio_rmb(vq) \
35 do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
36#define virtio_wmb(vq) \
37 do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
35#else 38#else
36/* We must force memory ordering even if guest is UP since host could be 39/* We must force memory ordering even if guest is UP since host could be
37 * running on another CPU, but SMP barriers are defined to barrier() in that 40 * running on another CPU, but SMP barriers are defined to barrier() in that
38 * configuration. So fall back to mandatory barriers instead. */ 41 * configuration. So fall back to mandatory barriers instead. */
39#define virtio_mb() mb() 42#define virtio_mb(vq) mb()
40#define virtio_rmb() rmb() 43#define virtio_rmb(vq) rmb()
41#define virtio_wmb() wmb() 44#define virtio_wmb(vq) wmb()
42#endif 45#endif
43 46
44#ifdef DEBUG 47#ifdef DEBUG
@@ -77,6 +80,9 @@ struct vring_virtqueue
77 /* Actual memory layout for this queue */ 80 /* Actual memory layout for this queue */
78 struct vring vring; 81 struct vring vring;
79 82
83 /* Can we use weak barriers? */
84 bool weak_barriers;
85
80 /* Other side has made a mess, don't try any more. */ 86 /* Other side has made a mess, don't try any more. */
81 bool broken; 87 bool broken;
82 88
@@ -245,14 +251,14 @@ void virtqueue_kick(struct virtqueue *_vq)
245 START_USE(vq); 251 START_USE(vq);
246 /* Descriptors and available array need to be set before we expose the 252 /* Descriptors and available array need to be set before we expose the
247 * new available array entries. */ 253 * new available array entries. */
248 virtio_wmb(); 254 virtio_wmb(vq);
249 255
250 old = vq->vring.avail->idx; 256 old = vq->vring.avail->idx;
251 new = vq->vring.avail->idx = old + vq->num_added; 257 new = vq->vring.avail->idx = old + vq->num_added;
252 vq->num_added = 0; 258 vq->num_added = 0;
253 259
254 /* Need to update avail index before checking if we should notify */ 260 /* Need to update avail index before checking if we should notify */
255 virtio_mb(); 261 virtio_mb(vq);
256 262
257 if (vq->event ? 263 if (vq->event ?
258 vring_need_event(vring_avail_event(&vq->vring), new, old) : 264 vring_need_event(vring_avail_event(&vq->vring), new, old) :
@@ -314,7 +320,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
314 } 320 }
315 321
316 /* Only get used array entries after they have been exposed by host. */ 322 /* Only get used array entries after they have been exposed by host. */
317 virtio_rmb(); 323 virtio_rmb(vq);
318 324
319 i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id; 325 i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
320 *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len; 326 *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
@@ -337,7 +343,7 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
337 * the read in the next get_buf call. */ 343 * the read in the next get_buf call. */
338 if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) { 344 if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
339 vring_used_event(&vq->vring) = vq->last_used_idx; 345 vring_used_event(&vq->vring) = vq->last_used_idx;
340 virtio_mb(); 346 virtio_mb(vq);
341 } 347 }
342 348
343 END_USE(vq); 349 END_USE(vq);
@@ -366,7 +372,7 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
366 * entry. Always do both to keep code simple. */ 372 * entry. Always do both to keep code simple. */
367 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 373 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
368 vring_used_event(&vq->vring) = vq->last_used_idx; 374 vring_used_event(&vq->vring) = vq->last_used_idx;
369 virtio_mb(); 375 virtio_mb(vq);
370 if (unlikely(more_used(vq))) { 376 if (unlikely(more_used(vq))) {
371 END_USE(vq); 377 END_USE(vq);
372 return false; 378 return false;
@@ -393,7 +399,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
393 /* TODO: tune this threshold */ 399 /* TODO: tune this threshold */
394 bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4; 400 bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
395 vring_used_event(&vq->vring) = vq->last_used_idx + bufs; 401 vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
396 virtio_mb(); 402 virtio_mb(vq);
397 if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) { 403 if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
398 END_USE(vq); 404 END_USE(vq);
399 return false; 405 return false;
@@ -453,6 +459,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt);
453struct virtqueue *vring_new_virtqueue(unsigned int num, 459struct virtqueue *vring_new_virtqueue(unsigned int num,
454 unsigned int vring_align, 460 unsigned int vring_align,
455 struct virtio_device *vdev, 461 struct virtio_device *vdev,
462 bool weak_barriers,
456 void *pages, 463 void *pages,
457 void (*notify)(struct virtqueue *), 464 void (*notify)(struct virtqueue *),
458 void (*callback)(struct virtqueue *), 465 void (*callback)(struct virtqueue *),
@@ -476,6 +483,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int num,
476 vq->vq.vdev = vdev; 483 vq->vq.vdev = vdev;
477 vq->vq.name = name; 484 vq->vq.name = name;
478 vq->notify = notify; 485 vq->notify = notify;
486 vq->weak_barriers = weak_barriers;
479 vq->broken = false; 487 vq->broken = false;
480 vq->last_used_idx = 0; 488 vq->last_used_idx = 0;
481 vq->num_added = 0; 489 vq->num_added = 0;
diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h
index 36be0f6e18a9..e338730c2660 100644
--- a/include/linux/virtio_ring.h
+++ b/include/linux/virtio_ring.h
@@ -168,6 +168,7 @@ struct virtqueue;
168struct virtqueue *vring_new_virtqueue(unsigned int num, 168struct virtqueue *vring_new_virtqueue(unsigned int num,
169 unsigned int vring_align, 169 unsigned int vring_align,
170 struct virtio_device *vdev, 170 struct virtio_device *vdev,
171 bool weak_barriers,
171 void *pages, 172 void *pages,
172 void (*notify)(struct virtqueue *vq), 173 void (*notify)(struct virtqueue *vq),
173 void (*callback)(struct virtqueue *vq), 174 void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
index 669bcdd45805..953db2abf6b9 100644
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -214,6 +214,7 @@ void *virtqueue_detach_unused_buf(struct virtqueue *vq);
214struct virtqueue *vring_new_virtqueue(unsigned int num, 214struct virtqueue *vring_new_virtqueue(unsigned int num,
215 unsigned int vring_align, 215 unsigned int vring_align,
216 struct virtio_device *vdev, 216 struct virtio_device *vdev,
217 bool weak_barriers,
217 void *pages, 218 void *pages,
218 void (*notify)(struct virtqueue *vq), 219 void (*notify)(struct virtqueue *vq),
219 void (*callback)(struct virtqueue *vq), 220 void (*callback)(struct virtqueue *vq),
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
index 74d3331bdaf9..0740284396c1 100644
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -92,7 +92,8 @@ static void vq_info_add(struct vdev_info *dev, int num)
92 assert(r >= 0); 92 assert(r >= 0);
93 memset(info->ring, 0, vring_size(num, 4096)); 93 memset(info->ring, 0, vring_size(num, 4096));
94 vring_init(&info->vring, num, info->ring, 4096); 94 vring_init(&info->vring, num, info->ring, 4096);
95 info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring, 95 info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev,
96 true, info->ring,
96 vq_notify, vq_callback, "test"); 97 vq_notify, vq_callback, "test");
97 assert(info->vq); 98 assert(info->vq);
98 info->vq->priv = info; 99 info->vq->priv = info;