[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[libvirt] [RFC PATCH v2 2/3] qemu: RDMA migration support using 'rdma' URI



From: "Michael R. Hines" <mrhines us ibm com>

The switch from x-rdma => rdma has not yet happened,
but at least we can review the patch until it goes
through on qemu-devel.

USAGE: $ virsh migrate --live --migrateuri x-rdma:hostname domain qemu+ssh://hostname/system

Full documenation of the feature: http://wiki.qemu.org/Features/RDMALiveMigration

This patch includes mainly making all the locations in
libvirt where the 'tcp' string was hard-coded to be
more flexible to use more than one protocol.

Signed-off-by: Michael R. Hines <mrhines us ibm com>
---
 src/qemu/qemu_capabilities.c |  13 +++++
 src/qemu/qemu_capabilities.h |   1 +
 src/qemu/qemu_command.c      |   8 ++++
 src/qemu/qemu_migration.c    | 110 ++++++++++++++++++++++++++++++++++---------
 src/qemu/qemu_monitor.c      |   3 +-
 src/qemu/qemu_monitor.h      |   1 +
 src/util/viruri.c            |   7 ++-
 7 files changed, 119 insertions(+), 24 deletions(-)

diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c
index 548b988..d82b48c 100644
--- a/src/qemu/qemu_capabilities.c
+++ b/src/qemu/qemu_capabilities.c
@@ -243,6 +243,8 @@ VIR_ENUM_IMPL(virQEMUCaps, QEMU_CAPS_LAST,
               "virtio-mmio",
               "ich9-intel-hda",
               "kvm-pit-lost-tick-policy",
+
+              "migrate-qemu-rdma", /* 160 */
     );
 
 struct _virQEMUCaps {
@@ -906,6 +908,9 @@ virCapsPtr virQEMUCapsInit(virQEMUCapsCachePtr cache)
     virCapabilitiesAddHostMigrateTransport(caps,
                                            "tcp");
 
+    virCapabilitiesAddHostMigrateTransport(caps,
+                                           "rdma");
+
     /* QEMU can support pretty much every arch that exists,
      * so just probe for them all - we gracefully fail
      * if a qemu-system-$ARCH binary can't be found
@@ -1110,6 +1115,7 @@ virQEMUCapsComputeCmdFlags(const char *help,
      *  -incoming unix   (qemu >= 0.12.0)
      *  -incoming fd     (qemu >= 0.12.0)
      *  -incoming stdio  (all earlier kvm)
+     *  -incoming rdma   (qemu >= 2.0.0)
      *
      * NB, there was a pre-kvm-79 'tcp' support, but it
      * was broken, because it blocked the monitor console
@@ -1130,6 +1136,9 @@ virQEMUCapsComputeCmdFlags(const char *help,
         virQEMUCapsSet(qemuCaps, QEMU_CAPS_MIGRATE_KVM_STDIO);
     }
 
+    if (version >= 2000000)
+        virQEMUCapsSet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_RDMA);
+
     if (version >= 10000)
         virQEMUCapsSet(qemuCaps, QEMU_CAPS_0_10);
 
@@ -2561,6 +2570,10 @@ virQEMUCapsInitQMPMonitor(virQEMUCapsPtr qemuCaps,
     if (qemuCaps->version >= 1006000)
         virQEMUCapsSet(qemuCaps, QEMU_CAPS_DEVICE_VIDEO_PRIMARY);
 
+    if (qemuCaps->version >= 2000000)
+        virQEMUCapsSet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_RDMA);
+
+
     if (virQEMUCapsProbeQMPCommands(qemuCaps, mon) < 0)
         goto cleanup;
     if (virQEMUCapsProbeQMPEvents(qemuCaps, mon) < 0)
diff --git a/src/qemu/qemu_capabilities.h b/src/qemu/qemu_capabilities.h
index 02d47c6..3e78961 100644
--- a/src/qemu/qemu_capabilities.h
+++ b/src/qemu/qemu_capabilities.h
@@ -198,6 +198,7 @@ enum virQEMUCapsFlags {
     QEMU_CAPS_DEVICE_VIRTIO_MMIO = 157, /* -device virtio-mmio */
     QEMU_CAPS_DEVICE_ICH9_INTEL_HDA = 158, /* -device ich9-intel-hda */
     QEMU_CAPS_KVM_PIT_TICK_POLICY = 159, /* kvm-pit.lost_tick_policy */
+    QEMU_CAPS_MIGRATE_QEMU_RDMA  = 160, /* have qemu rdma migration */
 
     QEMU_CAPS_LAST,                   /* this must always be the last item */
 };
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 763417f..0d23d8b 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -9448,6 +9448,14 @@ qemuBuildCommandLine(virConnectPtr conn,
                 goto error;
             }
             virCommandAddArg(cmd, migrateFrom);
+        } else if (STRPREFIX(migrateFrom, "rdma")) {
+            if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_RDMA)) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                               "%s", _("RDMA migration is not supported with "
+                                       "this QEMU binary"));
+                goto error;
+            }
+            virCommandAddArg(cmd, migrateFrom);
         } else if (STREQ(migrateFrom, "stdio")) {
             if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) {
                 virCommandAddArgFormat(cmd, "fd:%d", migrateFd);
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index ef6f1c5..1e0f538 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -46,6 +46,7 @@
 #include "virerror.h"
 #include "viralloc.h"
 #include "virfile.h"
+#include "virprocess.h"
 #include "datatypes.h"
 #include "fdstream.h"
 #include "viruuid.h"
@@ -2163,6 +2164,7 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
                         virDomainDefPtr *def,
                         const char *origname,
                         virStreamPtr st,
+                        const char *protocol,
                         unsigned short port,
                         bool autoPort,
                         const char *listenAddress,
@@ -2275,6 +2277,16 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
             freeaddrinfo(info);
             hostIPv6Capable = true;
         }
+
+        /*
+         * RDMA (iWarp) until linux 3.11 is broken, need
+         * better host librdmacm IPv6 support detection.
+         * Disallow by default for now if RDMA.
+         */
+        if (hostIPv6Capable && strstr(protocol, "rdma")) {
+            hostIPv6Capable = false;
+        }
+
         if (!(qemuCaps = virQEMUCapsCacheLookupCopy(driver->qemuCapsCache,
                                                     (*def)->emulator)))
             goto cleanup;
@@ -2318,9 +2330,9 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
          * -incoming <IPv4 addr>:port or -incoming <hostname>:port
          */
         if ((encloseAddress &&
-             virAsprintf(&migrateFrom, "tcp:[%s]:%d", listenAddress, port) < 0) ||
+             virAsprintf(&migrateFrom, "%s:[%s]:%d", protocol, listenAddress, port) < 0) ||
             (!encloseAddress &&
-             virAsprintf(&migrateFrom, "tcp:%s:%d", listenAddress, port) < 0))
+             virAsprintf(&migrateFrom, "%s:%s:%d", protocol, listenAddress, port) < 0))
             goto cleanup;
     }
 
@@ -2507,7 +2519,7 @@ qemuMigrationPrepareTunnel(virQEMUDriverPtr driver,
 
     ret = qemuMigrationPrepareAny(driver, dconn, cookiein, cookieinlen,
                                   cookieout, cookieoutlen, def, origname,
-                                  st, 0, false, NULL, flags);
+                                  st, "tcp", 0, false, NULL, flags);
     return ret;
 }
 
@@ -2529,6 +2541,8 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver,
     unsigned short port = 0;
     bool autoPort = true;
     char *hostname = NULL;
+    const char *protocol = NULL;
+    char *well_formed_protocol = NULL;
     const char *p;
     char *uri_str = NULL;
     int ret = -1;
@@ -2577,21 +2591,37 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver,
         if (virAsprintf(uri_out, "tcp:%s:%d", hostname, port) < 0)
             goto cleanup;
     } else {
-        /* Check the URI starts with "tcp:".  We will escape the
+        char * protocol_save = NULL;
+        char * uri_save = NULL;
+
+        /* Check the URI starts with a valid prefix.  We will escape the
          * URI when passing it to the qemu monitor, so bad
          * characters in hostname part don't matter.
          */
-        if (!(p = STRSKIP(uri_in, "tcp:"))) {
-            virReportError(VIR_ERR_INVALID_ARG, "%s",
-                           _("only tcp URIs are supported for KVM/QEMU"
-                             " migrations"));
+
+        if (VIR_STRDUP(uri_save, uri_in) <= 0) {
             goto cleanup;
         }
 
-        /* Convert uri_in to well-formed URI with // after tcp: */
-        if (!(STRPREFIX(uri_in, "tcp://"))) {
+        protocol = strtok_r(uri_save, ":", &protocol_save);
+        VIR_FREE(uri_save);
+        if (protocol) {
+            if (virAsprintf(&well_formed_protocol, "%s://", protocol) < 0)
+                goto cleanup;
+        }
+
+        /* Make sure it's a valid protocol */
+        if (!(p = STRSKIP(uri_in, "tcp:")) &&
+            !(p = STRSKIP(uri_in, "rdma:"))) {
+            virReportError(VIR_ERR_INVALID_ARG, _("URI %s (%s) not supported"
+                            " for KVM/QEMU migrations"), protocol, uri_in);
+            goto cleanup;
+        }
+
+        /* Convert uri_in to well-formed URI with // after colon */
+        if (!(STRPREFIX(uri_in, well_formed_protocol))) {
             well_formed_uri = false;
-            if (virAsprintf(&uri_str, "tcp://%s", p) < 0)
+            if (virAsprintf(&uri_str, "%s://%s", protocol, p) < 0)
                 goto cleanup;
         }
 
@@ -2637,10 +2667,13 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver,
 
     ret = qemuMigrationPrepareAny(driver, dconn, cookiein, cookieinlen,
                                   cookieout, cookieoutlen, def, origname,
-                                  NULL, port, autoPort, listenAddress, flags);
+                                  NULL, protocol ? protocol : "tcp",
+                                  port, autoPort, listenAddress, flags);
 cleanup:
     virURIFree(uri);
     VIR_FREE(hostname);
+    VIR_FREE(protocol);
+    VIR_FREE(well_formed_protocol);
     if (ret != 0) {
         VIR_FREE(*uri_out);
         if (autoPort)
@@ -2844,6 +2877,7 @@ struct _qemuMigrationSpec {
     enum qemuMigrationDestinationType destType;
     union {
         struct {
+            const char *proto;
             const char *name;
             int port;
         } host;
@@ -3205,6 +3239,7 @@ qemuMigrationRun(virQEMUDriverPtr driver,
     switch (spec->destType) {
     case MIGRATION_DEST_HOST:
         ret = qemuMonitorMigrateToHost(priv->mon, migrate_flags,
+                                       spec->dest.host.proto,
                                        spec->dest.host.name,
                                        spec->dest.host.port);
         break;
@@ -3335,7 +3370,7 @@ cancel:
     goto cleanup;
 }
 
-/* Perform migration using QEMU's native TCP migrate support,
+/* Perform migration using QEMU's native migrate support,
  * not encrypted obviously
  */
 static int doNativeMigrate(virQEMUDriverPtr driver,
@@ -3353,7 +3388,11 @@ static int doNativeMigrate(virQEMUDriverPtr driver,
     qemuDomainObjPrivatePtr priv = vm->privateData;
     virURIPtr uribits = NULL;
     int ret = -1;
+    char *tmp = NULL;
     qemuMigrationSpec spec;
+    char *well_formed_proto = NULL;
+    char * protocol_save = NULL;
+    char * uri_save = NULL;
 
     VIR_DEBUG("driver=%p, vm=%p, uri=%s, cookiein=%s, cookieinlen=%d, "
               "cookieout=%p, cookieoutlen=%p, flags=%lx, resource=%lu, "
@@ -3362,20 +3401,44 @@ static int doNativeMigrate(virQEMUDriverPtr driver,
               cookieout, cookieoutlen, flags, resource,
               NULLSTR(graphicsuri));
 
-    if (STRPREFIX(uri, "tcp:") && !STRPREFIX(uri, "tcp://")) {
-        char *tmp;
-        /* HACK: source host generates bogus URIs, so fix them up */
-        if (virAsprintf(&tmp, "tcp://%s", uri + strlen("tcp:")) < 0)
+    ret = VIR_STRDUP(uri_save, uri);
+    if (ret <= 0) {
+        return -1;
+    }
+
+    spec.dest.host.proto = strtok_r(uri_save, ":", &protocol_save);
+    VIR_FREE(uri_save);
+
+    /* HACK: source host generates bogus URIs, so fix them up */
+    if (spec.dest.host.proto) {
+        ret = virAsprintf(&well_formed_proto, "%s://",
+                            spec.dest.host.proto);
+        if (ret < 0)
+            return ret;
+    }
+
+    /* HACK: source host generates bogus URIs, so fix them up */
+
+    if (!STRPREFIX(uri, well_formed_proto)) {
+        if (virAsprintf(&tmp, "%s://%s", spec.dest.host.proto,
+                        uri + strlen(spec.dest.host.proto) + 1) < 0)
             return -1;
-        uribits = virURIParse(tmp);
-        VIR_FREE(tmp);
     } else {
         uribits = virURIParse(uri);
     }
-    if (!uribits)
-        return -1;
 
-    if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD))
+    if (tmp) {
+        uribits = virURIParse(tmp);
+        VIR_FREE(tmp);
+    }
+
+    if (!uribits) {
+        ret = -1;
+        goto err;
+    }
+
+    if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) &&
+            !STREQ(spec.dest.host.proto, "rdma"))
         spec.destType = MIGRATION_DEST_CONNECT_HOST;
     else
         spec.destType = MIGRATION_DEST_HOST;
@@ -3392,6 +3455,9 @@ static int doNativeMigrate(virQEMUDriverPtr driver,
 
     virURIFree(uribits);
 
+err:
+    VIR_FREE(well_formed_proto);
+
     return ret;
 }
 
diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index 1514715..5a450e2 100644
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -2164,6 +2164,7 @@ int qemuMonitorMigrateToFd(qemuMonitorPtr mon,
 
 int qemuMonitorMigrateToHost(qemuMonitorPtr mon,
                              unsigned int flags,
+                             const char *proto,
                              const char *hostname,
                              int port)
 {
@@ -2179,7 +2180,7 @@ int qemuMonitorMigrateToHost(qemuMonitorPtr mon,
     }
 
 
-    if (virAsprintf(&uri, "tcp:%s:%d", hostname, port) < 0)
+    if (virAsprintf(&uri, "%s:%s:%d", proto, hostname, port) < 0)
         return -1;
 
     if (mon->json)
diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
index 27f9cb4..16b0b77 100644
--- a/src/qemu/qemu_monitor.h
+++ b/src/qemu/qemu_monitor.h
@@ -476,6 +476,7 @@ int qemuMonitorMigrateToFd(qemuMonitorPtr mon,
 
 int qemuMonitorMigrateToHost(qemuMonitorPtr mon,
                              unsigned int flags,
+                             const char *proto,
                              const char *hostname,
                              int port);
 
diff --git a/src/util/viruri.c b/src/util/viruri.c
index 35efad8..662029a 100644
--- a/src/util/viruri.c
+++ b/src/util/viruri.c
@@ -187,7 +187,12 @@ virURIParse(const char *uri)
         goto error;
 
     /* First check: does it even make sense to jump inside */
-    if (ret->server != NULL &&
+
+    /*
+     * IPv6 rdma over iwarp is broken in linux. Waiting for a
+     * fix on the kernel side...
+     */
+    if (ret->server != NULL && !STREQ(ret->scheme, "rdma") &&
         ret->server[0] == '[') {
         size_t length = strlen(ret->server);
 
-- 
1.8.1.2


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]