host->guest throughput with kvm-83-29-txtimer1.stats

guest->host host->guest || cpu utilization Mb/s per % CPU throughput || kvm-83-29-txtimer1.stats kvm-83-29-txtimer2.stats

From 1181f18afb761f011fdd3a4ce7e53e4c3e951f44 Mon Sep 17 00:00:00 2001
From: Mark McLoughlin 
Date: Wed, 15 Apr 2009 11:12:24 +0100
Subject: [PATCH 1/2] kvm: qemu: virtio-net: Misc tx mitigation improvements

Keep tx notifies disabled while we flush the queue and then
re-check the queue once we have re-enabled notifies. This
should reduce the number of vmexits with any significant
packet rate.

If we have already set up a tx timer and disabled notifies
but we still get a notify, the queue must be full and we
should flush ASAP. Signal a flush from the I/O thread using
a bottom half so that we don't pause the guest by flushing
in the vcpu thread.

When we flush some packets from the tx bottom half or timer,
re-schedule the bottom half to quickly come back and check
for more I/O before re-enabling notifies.

Signed-off-by: Mark McLoughlin 
---
 qemu/hw/virtio-net.c |   54 +++++++++++++++++++++++++++++++++++++++----------
 1 files changed, 42 insertions(+), 11 deletions(-)

diff --git a/qemu/hw/virtio-net.c b/qemu/hw/virtio-net.c
index 8efd438..2c65496 100644
--- a/qemu/hw/virtio-net.c
+++ b/qemu/hw/virtio-net.c
@@ -31,6 +31,8 @@ typedef struct VirtIONet
     VLANClientState *vc;
     QEMUTimer *tx_timer;
     int tx_timer_active;
+    QEMUBH *tx_bh;
+    int tx_bh_scheduled;
     int mergeable_rx_bufs;
 } VirtIONet;
 
@@ -290,9 +292,10 @@ static void virtio_net_receive(void *opaque, const uint8_t *buf, int size)
 }
 
 /* TX */
-static void virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
+static int virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq, int enable_notify)
 {
     VirtQueueElement elem;
+    int num_packets = 0;
 #ifdef TAP_VNET_HDR
     int has_vnet_hdr = tap_has_vnet_hdr(n->vc->vlan->first_client);
 #else
@@ -300,7 +303,7 @@ static void virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
 #endif
 
     if (!(n->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK))
-        return;
+        return num_packets;
 
     while (virtqueue_pop(vq, &elem)) {
         ssize_t len = 0;
@@ -334,18 +337,30 @@ static void virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
 
         virtqueue_push(vq, &elem, len);
         virtio_notify(&n->vdev, vq);
+
+        num_packets++;
+    }
+
+    if (enable_notify) {
+        virtio_queue_set_notification(vq, 1);
+        num_packets += virtio_net_flush_tx(n, vq, 0);
     }
+
+    return num_packets;
 }
 
 static void virtio_net_handle_tx(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIONet *n = to_virtio_net(vdev);
 
+    if (n->tx_bh_scheduled)
+        return;
+
     if (n->tx_timer_active) {
-        virtio_queue_set_notification(vq, 1);
         qemu_del_timer(n->tx_timer);
         n->tx_timer_active = 0;
-        virtio_net_flush_tx(n, vq);
+        qemu_bh_schedule(n->tx_bh);
+        n->tx_bh_scheduled = 1;
     } else {
         qemu_mod_timer(n->tx_timer,
                        qemu_get_clock(vm_clock) + TX_TIMER_INTERVAL);
@@ -364,8 +379,24 @@ static void virtio_net_tx_timer(void *opaque)
     if (!(n->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK))
         return;
 
-    virtio_queue_set_notification(n->tx_vq, 1);
-    virtio_net_flush_tx(n, n->tx_vq);
+    if (virtio_net_flush_tx(n, n->tx_vq, 1)) {
+        virtio_queue_set_notification(n->tx_vq, 0);
+        qemu_bh_schedule(n->tx_bh);
+        n->tx_bh_scheduled = 1;
+    }
+}
+
+static void virtio_net_tx_bh(void *opaque)
+{
+    VirtIONet *n = opaque;
+
+    n->tx_bh_scheduled = 0;
+
+    if (virtio_net_flush_tx(n, n->tx_vq, 1)) {
+        virtio_queue_set_notification(n->tx_vq, 0);
+        qemu_bh_schedule(n->tx_bh);
+        n->tx_bh_scheduled = 1;
+    }
 }
 
 static void virtio_net_save(QEMUFile *f, void *opaque)
@@ -375,7 +406,7 @@ static void virtio_net_save(QEMUFile *f, void *opaque)
     virtio_save(&n->vdev, f);
 
     qemu_put_buffer(f, n->mac, 6);
-    qemu_put_be32(f, n->tx_timer_active);
+    qemu_put_be32(f, n->tx_timer_active || n->tx_bh_scheduled);
     qemu_put_be32(f, n->mergeable_rx_bufs);
 
 #ifdef TAP_VNET_HDR
@@ -393,7 +424,7 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
     virtio_load(&n->vdev, f);
 
     qemu_get_buffer(f, n->mac, 6);
-    n->tx_timer_active = qemu_get_be32(f);
+    n->tx_bh_scheduled = qemu_get_be32(f);
     n->mergeable_rx_bufs = qemu_get_be32(f);
 
 #ifdef TAP_VNET_HDR
@@ -401,9 +432,8 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
         tap_using_vnet_hdr(n->vc->vlan->first_client, 1);
 #endif
 
-    if (n->tx_timer_active) {
-        qemu_mod_timer(n->tx_timer,
-                       qemu_get_clock(vm_clock) + TX_TIMER_INTERVAL);
+    if (n->tx_bh_scheduled) {
+        qemu_bh_schedule(n->tx_bh);
     }
 
     return 0;
@@ -440,6 +470,8 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
 
     n->tx_timer = qemu_new_timer(vm_clock, virtio_net_tx_timer, n);
     n->tx_timer_active = 0;
+    n->tx_bh = qemu_bh_new(virtio_net_tx_bh, n);
+    n->tx_bh_scheduled = 0;
     n->mergeable_rx_bufs = 0;
 
     register_savevm("virtio-net", virtio_net_id++, 2,
-- 
1.6.0.6