[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[libvirt] [RFC 7/7] migrate: add hostdev migrate status to support hostdev migration



we add a migrate status for hostdev to specify the device don't
need to initialze when VM startup, after migration end, we add
the migrate status hostdev, so can support hostdev migration.

Signed-off-by: Chen Fan <chen fan fnst cn fujitsu com>
---
 src/conf/domain_conf.c    |  3 ++
 src/conf/domain_conf.h    |  7 ++++
 src/qemu/qemu_command.c   |  3 ++
 src/qemu/qemu_driver.c    | 53 +--------------------------
 src/qemu/qemu_hotplug.c   |  8 +++--
 src/qemu/qemu_migration.c | 92 ++++++++++++++++++++++++++++++++++++++++++++---
 src/qemu/qemu_migration.h |  4 +++
 src/util/virhostdev.c     |  3 ++
 8 files changed, 114 insertions(+), 59 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 7d1cd3e..b56c6fa 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -3035,6 +3035,9 @@ virDomainDeviceInfoIterateInternal(virDomainDefPtr def,
     device.type = VIR_DOMAIN_DEVICE_HOSTDEV;
     for (i = 0; i < def->nhostdevs; i++) {
         device.data.hostdev = def->hostdevs[i];
+        if (device.data.hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+            continue;
+
         if (cb(def, &device, def->hostdevs[i]->info, opaque) < 0)
             return -1;
     }
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 723f07b..4b7b4c9 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -543,6 +543,12 @@ struct _virDomainHostdevCaps {
     } u;
 };
 
+typedef enum {
+    VIR_DOMAIN_HOSTDEV_STATE_DEFAULT,
+    VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE,
+
+    VIR_DOMAIN_HOSTDEV_STATE_LAST
+} virDomainHostdevState;
 
 /* basic device for direct passthrough */
 struct _virDomainHostdevDef {
@@ -559,6 +565,7 @@ struct _virDomainHostdevDef {
     } source;
     virDomainHostdevOrigStates origstates;
     virDomainDeviceInfoPtr info; /* Guest address */
+    int state;
 };
 
 
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index e7e0937..dc5245a 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -10365,6 +10365,9 @@ qemuBuildCommandLine(virConnectPtr conn,
         virDomainHostdevDefPtr hostdev = def->hostdevs[i];
         char *devstr;
 
+        if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+            continue;
+
         if (hostdev->info->bootIndex) {
             if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS ||
                 (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI &&
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 0ba9e4a..4724171 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -12353,57 +12353,6 @@ qemuDomainMigrateBegin3(virDomainPtr domain,
                               cookieout, cookieoutlen, flags);
 }
 
-static int
-qemuDomainRemovePciPassThruDevices(virConnectPtr conn,
-                                   virDomainObjPtr vm)
-{
-    virQEMUDriverPtr driver = conn->privateData;
-    virDomainDeviceDef dev;
-    virDomainDeviceDefPtr dev_copy = NULL;
-    virCapsPtr caps = NULL;
-    int ret = -1;
-    size_t i;
-
-    if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
-        goto cleanup;
-
-    if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT))
-        goto cleanup;
-
-    /* unplug passthrough bond device */
-    for (i = 0; i < vm->def->nhostdevs; i++) {
-        virDomainHostdevDefPtr hostdev = vm->def->hostdevs[i];
-
-        if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS &&
-            hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI &&
-            hostdev->source.subsys.u.pci.backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO &&
-            hostdev->source.subsys.u.pci.device == VIR_DOMAIN_HOSTDEV_PCI_DEVICE_BOND) {
-
-            dev.type = VIR_DOMAIN_DEVICE_HOSTDEV;
-            dev.data.hostdev = hostdev;
-
-            dev_copy = virDomainDeviceDefCopy(&dev, vm->def, caps, driver->xmlopt);
-            if (!dev_copy)
-                goto cleanup;
-
-            if (qemuDomainDetachHostDevice(driver, vm, dev_copy) < 0) {
-                virDomainDeviceDefFree(dev_copy);
-                goto cleanup;
-            }
-
-            virDomainDeviceDefFree(dev_copy);
-            if (qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0)
-                goto cleanup;
-        }
-    }
-
-    ret = 0;
-
- cleanup:
-    virObjectUnref(caps);
-
-    return ret;
-}
 
 static char *
 qemuDomainMigrateBegin3Params(virDomainPtr domain,
@@ -12740,7 +12689,7 @@ qemuDomainMigratePerform3Params(virDomainPtr dom,
         return -1;
     }
 
-    if (qemuDomainRemovePciPassThruDevices(dom->conn, vm) < 0) {
+    if (qemuDomainMigratePciPassThruDevices(driver, vm, false) < 0) {
         qemuDomObjEndAPI(&vm);
         return -1;
     }
diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c
index f07c54d..13a7338 100644
--- a/src/qemu/qemu_hotplug.c
+++ b/src/qemu/qemu_hotplug.c
@@ -1239,8 +1239,9 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
     unsigned int flags = 0;
 
-    if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0)
-        return -1;
+    if (hostdev->state != VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+        if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0)
+            return -1;
 
     if (!cfg->relaxedACS)
         flags |= VIR_HOSTDEV_STRICT_ACS_CHECK;
@@ -1344,7 +1345,8 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
     if (ret < 0)
         goto error;
 
-    vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
+    if (hostdev->state != VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+        vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
 
     VIR_FREE(devstr);
     VIR_FREE(configfd_name);
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index 9ea83df..291cb9f 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -2001,10 +2001,7 @@ qemuMigrationIsAllowed(virQEMUDriverPtr driver, virDomainObjPtr vm,
     for (i = 0; i < def->nhostdevs; i++) {
         virDomainHostdevDefPtr hostdev = def->hostdevs[i];
 
-        if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS &&
-            hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI &&
-            hostdev->source.subsys.u.pci.backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO &&
-            hostdev->source.subsys.u.pci.device == VIR_DOMAIN_HOSTDEV_PCI_DEVICE_BOND)
+        if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
             continue;
 
         if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS ||
@@ -2629,6 +2626,80 @@ qemuMigrationCleanup(virDomainObjPtr vm,
 }
 
 
+static void
+qemuMigrationSetStateForHostdev(virDomainDefPtr def,
+                                int state)
+{
+    virDomainHostdevDefPtr hostdev;
+    size_t i;
+
+    if (!def)
+        return;
+
+    for (i = 0; i < def->nhostdevs; i++) {
+        hostdev = def->hostdevs[i];
+
+        if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS &&
+            hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI &&
+            hostdev->source.subsys.u.pci.backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO &&
+            hostdev->source.subsys.u.pci.device == VIR_DOMAIN_HOSTDEV_PCI_DEVICE_BOND)
+            hostdev->state = state;
+    }
+}
+
+
+int
+qemuDomainMigratePciPassThruDevices(virQEMUDriverPtr driver,
+                                    virDomainObjPtr vm,
+                                    bool isPlug)
+{
+    virDomainDeviceDef dev;
+    virDomainDeviceDefPtr dev_copy = NULL;
+    virDomainHostdevDefPtr hostdev;
+    virCapsPtr caps = NULL;
+    int ret = -1;
+    int i;
+
+    if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
+        goto cleanup;
+
+    /* plug/unplug passthrough bond device */
+    for (i = vm->def->nhostdevs; i >= 0; i--) {
+        hostdev = vm->def->hostdevs[i];
+
+        if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE) {
+            if (!isPlug) {
+                dev.type = VIR_DOMAIN_DEVICE_HOSTDEV;
+                dev.data.hostdev = hostdev;
+
+                dev_copy = virDomainDeviceDefCopy(&dev, vm->def, caps, driver->xmlopt);
+                if (!dev_copy)
+                    goto cleanup;
+
+                if (qemuDomainDetachHostDevice(driver, vm, dev_copy) < 0) {
+                    virDomainDeviceDefFree(dev_copy);
+                    goto cleanup;
+                }
+                virDomainDeviceDefFree(dev_copy);
+            } else {
+                qemuMigrationSetStateForHostdev(vm->def, VIR_DOMAIN_HOSTDEV_STATE_DEFAULT);
+                if (qemuDomainAttachHostDevice(NULL, driver, vm, hostdev) < 0)
+                    goto cleanup;
+            }
+            if (qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0)
+                goto cleanup;
+        }
+    }
+
+    ret = 0;
+
+ cleanup:
+    virObjectUnref(caps);
+
+    return ret;
+}
+
+
 /* The caller is supposed to lock the vm and start a migration job. */
 static char
 *qemuMigrationBeginPhase(virQEMUDriverPtr driver,
@@ -2662,6 +2733,8 @@ static char
     if (priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_OUT)
         qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_BEGIN3);
 
+    qemuMigrationSetStateForHostdev(vm->def, VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE);
+
     if (!qemuMigrationIsAllowed(driver, vm, NULL, true, abort_on_error))
         goto cleanup;
 
@@ -2885,6 +2958,8 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
     if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
         goto cleanup;
 
+    qemuMigrationSetStateForHostdev(*def, VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE);
+
     if (!qemuMigrationIsAllowed(driver, NULL, *def, true, abort_on_error))
         goto cleanup;
 
@@ -5315,6 +5390,13 @@ qemuMigrationFinish(virQEMUDriverPtr driver,
             goto endjob;
         }
 
+        /* hotplug previous mark migrate hostdev */
+        if (qemuDomainMigratePciPassThruDevices(driver, vm, true) < 0) {
+            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                           _("passthrough for hostdev failed"));
+            goto endjob;
+        }
+
         /* Guest is successfully running, so cancel previous auto destroy */
         qemuProcessAutoDestroyRemove(driver, vm);
     } else if (!(flags & VIR_MIGRATE_OFFLINE)) {
@@ -5331,6 +5413,8 @@ qemuMigrationFinish(virQEMUDriverPtr driver,
         VIR_WARN("Unable to encode migration cookie");
 
  endjob:
+    qemuMigrationSetStateForHostdev(vm->def, VIR_DOMAIN_HOSTDEV_STATE_DEFAULT);
+
     qemuMigrationJobFinish(driver, vm);
     if (!vm->persistent && !virDomainObjIsActive(vm))
         qemuDomainRemoveInactive(driver, vm);
diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h
index 1726455..fa21752 100644
--- a/src/qemu/qemu_migration.h
+++ b/src/qemu/qemu_migration.h
@@ -177,4 +177,8 @@ int qemuMigrationToFile(virQEMUDriverPtr driver, virDomainObjPtr vm,
     ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(5)
     ATTRIBUTE_RETURN_CHECK;
 
+int qemuDomainMigratePciPassThruDevices(virQEMUDriverPtr driver,
+                                        virDomainObjPtr vm,
+                                        bool isPlug);
+
 #endif /* __QEMU_MIGRATION_H__ */
diff --git a/src/util/virhostdev.c b/src/util/virhostdev.c
index f583e54..4b6152a 100644
--- a/src/util/virhostdev.c
+++ b/src/util/virhostdev.c
@@ -206,6 +206,9 @@ virHostdevGetPCIHostDeviceList(virDomainHostdevDefPtr *hostdevs, int nhostdevs)
         virDomainHostdevSubsysPCIPtr pcisrc = &hostdev->source.subsys.u.pci;
         virPCIDevicePtr dev;
 
+        if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+            continue;
+
         if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
             continue;
         if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
-- 
1.9.3


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]