[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[libvirt] [PATCH v3 02/14] Add public APIs for post-copy migration



From: Cristian Klein <cristiklein gmail com>

To use post-copy one has to start the migration with
VIR_MIGRATE_POSTCOPY flag and, while migration is in progress, call
virDomainMigrateStartPostCopy() to switch from pre-copy to post-copy.

Signed-off-by: Cristian Klein <cristiklein gmail com>
Signed-off-by: Jiri Denemark <jdenemar redhat com>
---

Notes:
    Version 3:
    - VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED will only be emitted on
      the destination host
    
    Version 2:
    - POSTCOPY_AFTER_PRECOPY flag removed

 include/libvirt/libvirt-domain.h |   4 ++
 src/driver-hypervisor.h          |   5 ++
 src/libvirt-domain.c             | 141 ++++++++++++++++++++++++++++++++++++---
 src/libvirt_public.syms          |   4 ++
 src/remote/remote_driver.c       |   1 +
 src/remote/remote_protocol.x     |  13 +++-
 src/remote_protocol-structs      |   5 ++
 7 files changed, 164 insertions(+), 9 deletions(-)

diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h
index 3846ab9..dede3da 100644
--- a/include/libvirt/libvirt-domain.h
+++ b/include/libvirt/libvirt-domain.h
@@ -678,6 +678,7 @@ typedef enum {
     VIR_MIGRATE_ABORT_ON_ERROR    = (1 << 12), /* abort migration on I/O errors happened during migration */
     VIR_MIGRATE_AUTO_CONVERGE     = (1 << 13), /* force convergence */
     VIR_MIGRATE_RDMA_PIN_ALL      = (1 << 14), /* RDMA memory pinning */
+    VIR_MIGRATE_POSTCOPY          = (1 << 15), /* enable (but do not start) post-copy migration */
 } virDomainMigrateFlags;
 
 
@@ -826,6 +827,9 @@ int virDomainMigrateGetMaxSpeed(virDomainPtr domain,
                                 unsigned long *bandwidth,
                                 unsigned int flags);
 
+int virDomainMigrateStartPostCopy(virDomainPtr domain,
+                                  unsigned int flags);
+
 char * virConnectGetDomainCapabilities(virConnectPtr conn,
                                        const char *emulatorbin,
                                        const char *arch,
diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h
index 2cbd01b..9bc3211 100644
--- a/src/driver-hypervisor.h
+++ b/src/driver-hypervisor.h
@@ -638,6 +638,10 @@ typedef int
                                     const char *dom_xml);
 
 typedef int
+(*virDrvDomainMigrateStartPostCopy)(virDomainPtr domain,
+                                    unsigned int flags);
+
+typedef int
 (*virDrvConnectIsEncrypted)(virConnectPtr conn);
 
 typedef int
@@ -1455,6 +1459,7 @@ struct _virHypervisorDriver {
     virDrvDomainSetUserPassword domainSetUserPassword;
     virDrvConnectRegisterCloseCallback connectRegisterCloseCallback;
     virDrvConnectUnregisterCloseCallback connectUnregisterCloseCallback;
+    virDrvDomainMigrateStartPostCopy domainMigrateStartPostCopy;
 };
 
 
diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c
index ca32dc1..1b95664 100644
--- a/src/libvirt-domain.c
+++ b/src/libvirt-domain.c
@@ -3537,6 +3537,7 @@ virDomainMigrateUnmanaged(virDomainPtr domain,
  *                                 automatically when supported).
  *   VIR_MIGRATE_UNSAFE    Force migration even if it is considered unsafe.
  *   VIR_MIGRATE_OFFLINE Migrate offline
+ *   VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy
  *
  * VIR_MIGRATE_TUNNELLED requires that VIR_MIGRATE_PEER2PEER be set.
  * Applications using the VIR_MIGRATE_PEER2PEER flag will probably
@@ -3573,6 +3574,11 @@ virDomainMigrateUnmanaged(virDomainPtr domain,
  * not support this feature and will return an error if bandwidth
  * is not 0.
  *
+ * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy
+ * migration.  Use virDomainMigrateStartPostCopy to switch migration into
+ * the post-copy mode.  See virDomainMigrateStartPostCopy for more details
+ * about post-copy.
+ *
  * To see which features are supported by the current hypervisor,
  * see virConnectGetCapabilities, /capabilities/host/migration_features.
  *
@@ -3748,6 +3754,7 @@ virDomainMigrate(virDomainPtr domain,
  *                                 automatically when supported).
  *   VIR_MIGRATE_UNSAFE    Force migration even if it is considered unsafe.
  *   VIR_MIGRATE_OFFLINE Migrate offline
+ *   VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy
  *
  * VIR_MIGRATE_TUNNELLED requires that VIR_MIGRATE_PEER2PEER be set.
  * Applications using the VIR_MIGRATE_PEER2PEER flag will probably
@@ -3784,6 +3791,11 @@ virDomainMigrate(virDomainPtr domain,
  * not support this feature and will return an error if bandwidth
  * is not 0.
  *
+ * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy
+ * migration.  Use virDomainMigrateStartPostCopy to switch migration into
+ * the post-copy mode.  See virDomainMigrateStartPostCopy for more details
+ * about post-copy.
+ *
  * To see which features are supported by the current hypervisor,
  * see virConnectGetCapabilities, /capabilities/host/migration_features.
  *
@@ -3968,6 +3980,11 @@ virDomainMigrate2(virDomainPtr domain,
  * can use either VIR_MIGRATE_NON_SHARED_DISK or
  * VIR_MIGRATE_NON_SHARED_INC as they are mutually exclusive.
  *
+ * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy
+ * migration.  Use virDomainMigrateStartPostCopy to switch migration into
+ * the post-copy mode.  See virDomainMigrateStartPostCopy for more details
+ * about post-copy.
+ *
  * There are many limitations on migration imposed by the underlying
  * technology - for example it may not be possible to migrate between
  * different processors even with the same architecture, or between
@@ -4208,6 +4225,7 @@ int virDomainMigrateUnmanagedCheckCompat(virDomainPtr domain,
  *                                 automatically when supported).
  *   VIR_MIGRATE_UNSAFE    Force migration even if it is considered unsafe.
  *   VIR_MIGRATE_OFFLINE Migrate offline
+ *   VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy
  *
  * The operation of this API hinges on the VIR_MIGRATE_PEER2PEER flag.
  * If the VIR_MIGRATE_PEER2PEER flag is NOT set, the duri parameter
@@ -4240,6 +4258,11 @@ int virDomainMigrateUnmanagedCheckCompat(virDomainPtr domain,
  * not support this feature and will return an error if bandwidth
  * is not 0.
  *
+ * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy
+ * migration.  Use virDomainMigrateStartPostCopy to switch migration into
+ * the post-copy mode.  See virDomainMigrateStartPostCopy for more details
+ * about post-copy.
+ *
  * To see which features are supported by the current hypervisor,
  * see virConnectGetCapabilities, /capabilities/host/migration_features.
  *
@@ -4321,6 +4344,7 @@ virDomainMigrateToURI(virDomainPtr domain,
  *                                 automatically when supported).
  *   VIR_MIGRATE_UNSAFE    Force migration even if it is considered unsafe.
  *   VIR_MIGRATE_OFFLINE Migrate offline
+ *   VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy
  *
  * The operation of this API hinges on the VIR_MIGRATE_PEER2PEER flag.
  *
@@ -4366,6 +4390,11 @@ virDomainMigrateToURI(virDomainPtr domain,
  * not support this feature and will return an error if bandwidth
  * is not 0.
  *
+ * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy
+ * migration.  Use virDomainMigrateStartPostCopy to switch migration into
+ * the post-copy mode.  See virDomainMigrateStartPostCopy for more details
+ * about post-copy.
+ *
  * To see which features are supported by the current hypervisor,
  * see virConnectGetCapabilities, /capabilities/host/migration_features.
  *
@@ -4446,6 +4475,11 @@ virDomainMigrateToURI2(virDomainPtr domain,
  * can use either VIR_MIGRATE_NON_SHARED_DISK or
  * VIR_MIGRATE_NON_SHARED_INC as they are mutually exclusive.
  *
+ * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy
+ * migration.  Use virDomainMigrateStartPostCopy to switch migration into
+ * the post-copy mode.  See virDomainMigrateStartPostCopy for more details
+ * about post-copy.
+ *
  * There are many limitations on migration imposed by the underlying
  * technology - for example it may not be possible to migrate between
  * different processors even with the same architecture, or between
@@ -8868,14 +8902,15 @@ virDomainGetJobInfo(virDomainPtr domain, virDomainJobInfoPtr info)
  *
  * When @flags contains VIR_DOMAIN_JOB_STATS_COMPLETED, the function will
  * return statistics about a recently completed job. Specifically, this
- * flag may be used to query statistics of a completed incoming migration.
- * Statistics of a completed job are automatically destroyed once read or
- * when libvirtd is restarted. Note that time information returned for
- * completed migrations may be completely irrelevant unless both source and
- * destination hosts have synchronized time (i.e., NTP daemon is running on
- * both of them). The statistics of a completed job can also be obtained by
- * listening to a VIR_DOMAIN_EVENT_ID_JOB_COMPLETED event (on the source host
- * in case of a migration job).
+ * flag may be used to query statistics of a completed incoming pre-copy
+ * migration (statistics for post-copy migration are only available on the
+ * source hsot). Statistics of a completed job are automatically destroyed
+ * once read or when libvirtd is restarted. Note that time information
+ * returned for completed migrations may be completely irrelevant unless both
+ * source and destination hosts have synchronized time (i.e., NTP daemon is
+ * running on both of them). The statistics of a completed job can also be
+ * obtained by listening to a VIR_DOMAIN_EVENT_ID_JOB_COMPLETED event (on the
+ * source host in case of a migration job).
  *
  * Returns 0 in case of success and -1 in case of failure.
  */
@@ -9165,6 +9200,96 @@ virDomainMigrateGetMaxSpeed(virDomainPtr domain,
 
 
 /**
+ * virDomainMigrateStartPostCopy:
+ * @domain: a domain object
+ * @flags: extra flags; not used yet, so callers should always pass 0
+ *
+ * Starts post-copy migration. This function has to be called while
+ * migration (initiated with VIR_MIGRATE_POSTCOPY flag) is in progress.
+ *
+ * Traditional post-copy migration iteratively walks through guest memory
+ * pages and migrates those that changed since the previous iteration. The
+ * iterative phase stops when the number of dirty pages is low enough so that
+ * the virtual CPUs can be paused, all dirty pages transferred to the
+ * destination, where the virtual CPUs are unpaused, and all this can happen
+ * within a predefined downtime period. It's clear that this process may never
+ * converge if downtime is too short and/or the guest keeps changing a lot of
+ * memory pages.
+ *
+ * When migration is switched to post-copy mode, the virtual CPUs are paused
+ * immediately, only a minimum set of pages is transferred, and the CPUs are
+ * unpaused on destination. The source keeps sending all remaining memory pages
+ * to the destination while the guest is already running there. Whenever the
+ * guest tries to read a memory page which has not been migrated yet, the
+ * hypervisor has to tell the source to transfer that page in a priority
+ * channel. To minimize such page faults, it is a good idea to run at least one
+ * iteration of pre-copy migration before switching to post-copy.
+ *
+ * Post-copy migration is guaranteed to converge since each page is transferred
+ * at most once no matter how fast it changes. On the other hand once the
+ * guest is running on the destination host, the migration can no longer be
+ * rolled back because none of the hosts has complete state. If this happens,
+ * libvirt will leave the domain paused on both hosts with
+ * VIR_DOMAIN_PAUSED_POSTCOPY_FAILED reason. It's up to the upper layer to
+ * decide what to do in such case.
+ *
+ * The following domain life cycle events are emitted during post-copy
+ * migration:
+ *  VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY (on the source) -- migration entered
+ *      post-copy mode.
+ *  VIR_DOMAIN_EVENT_RESUMED_POSTCOPY (on the destination) -- the guest is
+ *      running on the destination host while some of its memory pages still
+ *      remain on the source host; neither the source nor the destination host
+ *      contain a complete guest state from this point until migration
+ *      finishes.
+ *  VIR_DOMAIN_EVENT_RESUMED_MIGRATED (on the destination),
+ *  VIR_DOMAIN_EVENT_STOPPED_MIGRATED (on the source) -- migration finished
+ *      successfully and the destination host holds a complete guest state.
+ *  VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED (on the destination) -- emitted
+ *      when migration fails in post-copy mode and it's unclear whether any
+ *      of the hosts has a complete guest state.
+ *
+ * The progress of a post-copy migration can be monitored normally using
+ * virDomainGetJobStats on the source host. Fetching statistics of a completed
+ * post-copy migration can also be done on the source host (by calling
+ * virDomainGetJobStats or listening to VIR_DOMAIN_EVENT_ID_JOB_COMPLETED
+ * event, but (in contrast to pre-copy migration) the statistics are not
+ * available on the destination host. Thus, VIR_DOMAIN_EVENT_ID_JOB_COMPLETED
+ * event is the only way of getting statistics of a completed post-copy
+ * migration of a transient domain (because the domain is removed after
+ * migration and there's no domain to run virDomainGetJobStats on).
+ *
+ * Returns 0 in case of success, -1 otherwise.
+ */
+int
+virDomainMigrateStartPostCopy(virDomainPtr domain,
+                              unsigned int flags)
+{
+    virConnectPtr conn;
+
+    VIR_DOMAIN_DEBUG(domain);
+
+    virResetLastError();
+
+    virCheckDomainReturn(domain, -1);
+    conn = domain->conn;
+
+    virCheckReadOnlyGoto(conn->flags, error);
+
+    if (conn->driver->domainMigrateStartPostCopy) {
+        if (conn->driver->domainMigrateStartPostCopy(domain, flags) < 0)
+            goto error;
+        return 0;
+    }
+
+    virReportUnsupportedError();
+ error:
+    virDispatchError(conn);
+    return -1;
+}
+
+
+/**
  * virConnectDomainEventRegisterAny:
  * @conn: pointer to the connection
  * @dom: pointer to the domain
diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms
index dd94191..d98414d 100644
--- a/src/libvirt_public.syms
+++ b/src/libvirt_public.syms
@@ -725,4 +725,8 @@ LIBVIRT_1.2.19 {
         virDomainRename;
 } LIBVIRT_1.2.17;
 
+LIBVIRT_1.3.2 {
+        virDomainMigrateStartPostCopy;
+} LIBVIRT_1.2.19;
+
 # .... define new API here using predicted next version number ....
diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c
index 2daa507..09de5f8 100644
--- a/src/remote/remote_driver.c
+++ b/src/remote/remote_driver.c
@@ -8417,6 +8417,7 @@ static virHypervisorDriver hypervisor_driver = {
     .domainRename = remoteDomainRename, /* 1.2.19 */
     .connectRegisterCloseCallback = remoteConnectRegisterCloseCallback, /* 1.3.2 */
     .connectUnregisterCloseCallback = remoteConnectUnregisterCloseCallback, /* 1.3.2 */
+    .domainMigrateStartPostCopy = remoteDomainMigrateStartPostCopy, /* 1.3.3 */
 };
 
 static virNetworkDriver network_driver = {
diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x
index 952686c..6adad7a 100644
--- a/src/remote/remote_protocol.x
+++ b/src/remote/remote_protocol.x
@@ -3238,6 +3238,11 @@ struct remote_domain_event_callback_job_completed_msg {
     remote_typed_param params<REMOTE_DOMAIN_JOB_STATS_MAX>;
 };
 
+struct remote_domain_migrate_start_post_copy_args {
+    remote_nonnull_domain dom;
+    unsigned int flags;
+};
+
 /*----- Protocol. -----*/
 
 /* Define the program number, protocol version and procedure numbers here. */
@@ -5740,5 +5745,11 @@ enum remote_procedure {
      * @generate: both
      * @acl: none
      */
-    REMOTE_PROC_DOMAIN_EVENT_CALLBACK_JOB_COMPLETED = 363
+    REMOTE_PROC_DOMAIN_EVENT_CALLBACK_JOB_COMPLETED = 363,
+
+    /**
+     * @generate: both
+     * @acl: domain:migrate
+     */
+    REMOTE_PROC_DOMAIN_MIGRATE_START_POST_COPY = 364
 };
diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs
index 070338c..9d59e0e 100644
--- a/src/remote_protocol-structs
+++ b/src/remote_protocol-structs
@@ -2708,6 +2708,10 @@ struct remote_domain_event_callback_job_completed_msg {
                 remote_typed_param * params_val;
         } params;
 };
+struct remote_domain_migrate_start_post_copy_args {
+        remote_nonnull_domain      dom;
+        u_int                      flags;
+};
 enum remote_procedure {
         REMOTE_PROC_CONNECT_OPEN = 1,
         REMOTE_PROC_CONNECT_CLOSE = 2,
@@ -3072,4 +3076,5 @@ enum remote_procedure {
         REMOTE_PROC_CONNECT_CLOSE_CALLBACK_UNREGISTER = 361,
         REMOTE_PROC_CONNECT_EVENT_CONNECTION_CLOSED = 362,
         REMOTE_PROC_DOMAIN_EVENT_CALLBACK_JOB_COMPLETED = 363,
+        REMOTE_PROC_DOMAIN_MIGRATE_START_POST_COPY = 364,
 };
-- 
2.7.2


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]