[libvirt] [PATCH v2 3/3] libxl: add tunnelled migration support
Jim Fehlig
jfehlig at suse.com
Tue Feb 14 03:34:27 UTC 2017
On 02/07/2017 05:35 PM, Joao Martins wrote:
> From: Bob Liu <bob.liu at oracle.com>
>
> Tunnelled migration doesn't require any extra network connections beside
> the libvirt daemon. It's capable of strong encryption and the default
> option of openstack-nova.
>
> This patch adds the tunnelled migration(Tunnel3params) support to libxl.
> On the source side, the data flow is:
>
> * libxlDoMigrateSend() -> pipe libxlTunnel3MigrationFunc() polls pipe
> * out and then write to dest stream.
>
> While on the destination side:
> * Stream -> pipe -> 'recvfd of libxlDomainStartRestore'
>
> The usage is the same as p2p migration, execpt adding one extra
> '--tunnelled' to the libvirt p2p migration command.
>
> Signed-off-by: Bob Liu <bob.liu at oracle.com>
> Signed-off-by: Joao Martins <joao.m.martins at oracle.com>
> ---
> v2 (comments from Jim):
> * Adjust common preparetunnel function reflecting v1 suggestions
> - Using common top-level Prepare function by adding is_tunnel variable
> - Using libxl_migration PrepareAny added in patch 1
> - virHook support in PrepareTunnel3
> * Move virThreadPtr from libxlTunnelMigrationThread into libxlTunnelControl
> * Rename function local variable from tmThreadPtr to arg
> * Remove redundant assignment "tmThreadPtr = &tc->tmThread;" always being not
> null.
> ---
> src/libxl/libxl_driver.c | 49 ++++++--
> src/libxl/libxl_migration.c | 280 +++++++++++++++++++++++++++++++++++++++++---
> src/libxl/libxl_migration.h | 9 ++
> 3 files changed, 313 insertions(+), 25 deletions(-)
>
> diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c
> index 7bc8adf..b34f120 100644
> --- a/src/libxl/libxl_driver.c
> +++ b/src/libxl/libxl_driver.c
> @@ -5938,14 +5938,14 @@ libxlDomainMigratePrepareCommon(virConnectPtr dconn,
> char **cookieout ATTRIBUTE_UNUSED,
> int *cookieoutlen ATTRIBUTE_UNUSED,
> unsigned int flags,
> - void *data)
> + void *data,
> + bool is_tunnel)
> {
> libxlDriverPrivatePtr driver = dconn->privateData;
> virDomainDefPtr def = NULL;
> const char *dom_xml = NULL;
> const char *dname = NULL;
> const char *uri_in = NULL;
> - char **uri_out = data;
>
> #ifdef LIBXL_HAVE_NO_SUSPEND_RESUME
> virReportUnsupportedError();
> @@ -5971,12 +5971,25 @@ libxlDomainMigratePrepareCommon(virConnectPtr dconn,
> if (!(def = libxlDomainMigrationPrepareDef(driver, dom_xml, dname)))
> goto error;
>
> - if (virDomainMigratePrepare3ParamsEnsureACL(dconn, def) < 0)
> - goto error;
> + if (is_tunnel) {
> + virStreamPtr st = data;
>
> - if (libxlDomainMigrationPrepare(dconn, &def, uri_in, uri_out,
> - cookiein, cookieinlen, flags) < 0)
> - goto error;
> + if (virDomainMigratePrepareTunnel3ParamsEnsureACL(dconn, def) < 0)
> + goto error;
> +
> + if (libxlDomainMigrationPrepareTunnel3(dconn, st, &def, cookiein,
> + cookieinlen, flags) < 0)
> + goto error;
> + } else {
> + char **uri_out = data;
> +
> + if (virDomainMigratePrepare3ParamsEnsureACL(dconn, def) < 0)
> + goto error;
> +
> + if (libxlDomainMigrationPrepare(dconn, &def, uri_in, uri_out,
> + cookiein, cookieinlen, flags) < 0)
> + goto error;
> + }
Same comment here about the ACL checks and 'make check. With both patches applied
./libxl/libxl_driver.c:5981 Mismatch check
'virDomainMigratePrepareTunnel3ParamsEnsureACL' for function
'libxlDomainMigratePrepareCommon'
./libxl/libxl_driver.c:5990 Mismatch check
'virDomainMigratePrepare3ParamsEnsureACL' for function
'libxlDomainMigratePrepareCommon'
>
> return 0;
>
> @@ -5999,7 +6012,24 @@ libxlDomainMigratePrepare3Params(virConnectPtr dconn,
> return libxlDomainMigratePrepareCommon(dconn, params, nparams,
> cookiein, cookieinlen,
> cookieout, cookieoutlen,
> - flags, uri_out);
> + flags, uri_out, false);
> +}
> +
> +static int
> +libxlDomainMigratePrepareTunnel3Params(virConnectPtr dconn,
> + virStreamPtr st,
> + virTypedParameterPtr params,
> + int nparams,
> + const char *cookiein,
> + int cookieinlen,
> + char **cookieout ATTRIBUTE_UNUSED,
> + int *cookieoutlen ATTRIBUTE_UNUSED,
> + unsigned int flags)
> +{
> + return libxlDomainMigratePrepareCommon(dconn, params, nparams,
> + cookiein, cookieinlen,
> + cookieout, cookieoutlen,
> + flags, st, true);
> }
>
> static int
> @@ -6047,7 +6077,7 @@ libxlDomainMigratePerform3Params(virDomainPtr dom,
> if (virDomainMigratePerform3ParamsEnsureACL(dom->conn, vm->def) < 0)
> goto cleanup;
>
> - if (flags & VIR_MIGRATE_PEER2PEER) {
> + if ((flags & (VIR_MIGRATE_TUNNELLED | VIR_MIGRATE_PEER2PEER))) {
> if (libxlDomainMigrationPerformP2P(driver, vm, dom->conn, dom_xml,
> dconnuri, uri, dname, flags) < 0)
> goto cleanup;
> @@ -6532,6 +6562,7 @@ static virHypervisorDriver libxlHypervisorDriver = {
> .nodeDeviceReset = libxlNodeDeviceReset, /* 1.2.3 */
> .domainMigrateBegin3Params = libxlDomainMigrateBegin3Params, /* 1.2.6 */
> .domainMigratePrepare3Params = libxlDomainMigratePrepare3Params, /* 1.2.6 */
> + .domainMigratePrepareTunnel3Params = libxlDomainMigratePrepareTunnel3Params, /* 3.1.0 */
> .domainMigratePerform3Params = libxlDomainMigratePerform3Params, /* 1.2.6 */
> .domainMigrateFinish3Params = libxlDomainMigrateFinish3Params, /* 1.2.6 */
> .domainMigrateConfirm3Params = libxlDomainMigrateConfirm3Params, /* 1.2.6 */
> diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c
> index 7beabd2..3aa0797 100644
> --- a/src/libxl/libxl_migration.c
> +++ b/src/libxl/libxl_migration.c
> @@ -44,6 +44,7 @@
> #include "libxl_migration.h"
> #include "locking/domain_lock.h"
> #include "virtypedparam.h"
> +#include "fdstream.h"
>
> #define VIR_FROM_THIS VIR_FROM_LIBXL
>
> @@ -554,6 +555,95 @@ libxlDomainMigrationPrepareAny(virConnectPtr dconn,
> }
>
> int
> +libxlDomainMigrationPrepareTunnel3(virConnectPtr dconn,
> + virStreamPtr st,
> + virDomainDefPtr *def,
> + const char *cookiein,
> + int cookieinlen,
> + unsigned int flags)
> +{
> + libxlMigrationCookiePtr mig = NULL;
> + libxlDriverPrivatePtr driver = dconn->privateData;
> + virDomainObjPtr vm = NULL;
> + libxlMigrationDstArgs *args = NULL;
> + virThread thread;
> + bool taint_hook = false;
> + libxlDomainObjPrivatePtr priv = NULL;
> + char *xmlout = NULL;
> + int dataFD[2] = { -1, -1 };
> + int ret = -1;
> +
> + if (libxlDomainMigrationPrepareAny(dconn, def, cookiein, cookieinlen,
> + &mig, &xmlout, &taint_hook) < 0)
> + goto error;
> +
> + if (!(vm = virDomainObjListAdd(driver->domains, *def,
> + driver->xmlopt,
> + VIR_DOMAIN_OBJ_LIST_ADD_LIVE |
> + VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE,
> + NULL)))
> + goto error;
> +
> + *def = NULL;
> + priv = vm->privateData;
> +
> + if (taint_hook) {
> + /* Domain XML has been altered by a hook script. */
> + priv->hookRun = true;
> + }
> +
> + /*
> + * The data flow of tunnel3 migration in the dest side:
> + * stream -> pipe -> recvfd of libxlDomainStartRestore
> + */
> + if (pipe(dataFD) < 0)
> + goto error;
> +
> + /* Stream data will be written to pipeIn */
> + if (virFDStreamOpen(st, dataFD[1]) < 0)
> + goto error;
> + dataFD[1] = -1; /* 'st' owns the FD now & will close it */
> +
> + if (libxlMigrationDstArgsInitialize() < 0)
> + goto error;
> +
> + if (!(args = virObjectNew(libxlMigrationDstArgsClass)))
> + goto error;
> +
> + args->conn = dconn;
> + args->vm = vm;
> + args->flags = flags;
> + args->migcookie = mig;
> + /* Receive from pipeOut */
> + args->recvfd = dataFD[0];
> + args->nsocks = 0;
> + if (virThreadCreate(&thread, false, libxlDoMigrateReceive, args) < 0) {
> + virReportError(VIR_ERR_OPERATION_FAILED, "%s",
> + _("Failed to create thread for receiving migration data"));
> + goto error;
> + }
> +
> + ret = 0;
> + goto done;
> +
> + error:
> + VIR_FORCE_CLOSE(dataFD[1]);
> + VIR_FORCE_CLOSE(dataFD[0]);
> + virObjectUnref(args);
> + /* Remove virDomainObj from domain list */
> + if (vm) {
> + virDomainObjListRemove(driver->domains, vm);
> + vm = NULL;
> + }
> +
> + done:
> + if (vm)
> + virObjectUnlock(vm);
> +
> + return ret;
> +}
> +
> +int
> libxlDomainMigrationPrepare(virConnectPtr dconn,
> virDomainDefPtr *def,
> const char *uri_in,
> @@ -734,9 +824,148 @@ libxlDomainMigrationPrepare(virConnectPtr dconn,
> return ret;
> }
>
> -/* This function is a simplification of virDomainMigrateVersion3Full
> - * excluding tunnel support and restricting it to migration v3
> - * with params since it was the first to be introduced in libxl.
> +typedef struct _libxlTunnelMigrationThread libxlTunnelMigrationThread;
> +struct _libxlTunnelMigrationThread {
> + virStreamPtr st;
> + int srcFD;
> +};
> +#define TUNNEL_SEND_BUF_SIZE 65536
> +
> +/*
> + * The data flow of tunnel3 migration in the src side:
> + * libxlDoMigrateSend() -> pipe
> + * libxlTunnel3MigrationFunc() polls pipe out and then write to dest stream.
> + */
> +static void libxlTunnel3MigrationFunc(void *arg)
> +{
> + libxlTunnelMigrationThread *data = (libxlTunnelMigrationThread *)arg;
> + char *buffer = NULL;
> + struct pollfd fds[1];
> + int timeout = -1;
> +
> + if (VIR_ALLOC_N(buffer, TUNNEL_SEND_BUF_SIZE) < 0) {
> + virReportError(errno, "%s", _("poll failed in migration tunnel"));
Error reporting is handled by VIR_ALLOC_N.
> + return;
> + }
> +
> + fds[0].fd = data->srcFD;
> + for (;;) {
> + int ret;
> +
> + fds[0].events = POLLIN;
> + fds[0].revents = 0;
> + ret = poll(fds, ARRAY_CARDINALITY(fds), timeout);
> + if (ret < 0) {
> + if (errno == EAGAIN || errno == EINTR)
> + continue;
> + virReportError(errno, "%s",
> + _("poll failed in libxlTunnel3MigrationFunc"));
> + goto cleanup;
> + }
> +
> + if (ret == 0) {
> + VIR_DEBUG("poll got timeout");
> + break;
> + }
> +
> + if (fds[0].revents & (POLLIN | POLLERR | POLLHUP)) {
> + int nbytes;
> +
> + nbytes = read(data->srcFD, buffer, TUNNEL_SEND_BUF_SIZE);
> + if (nbytes > 0) {
> + /* Write to dest stream */
> + if (virStreamSend(data->st, buffer, nbytes) < 0) {
> + virReportError(errno, "%s",
> + _("tunnelled migration failed to send to virStream"));
virStreamSend also reports (a more helpful?) error.
> + virStreamAbort(data->st);
> + goto cleanup;
> + }
> + } else if (nbytes < 0) {
> + virReportError(errno, "%s",
> + _("tunnelled migration failed to read from xen side"));
> + virStreamAbort(data->st);
> + goto cleanup;
> + } else {
> + /* EOF; transferred all data */
> + break;
> + }
> + }
> + }
> +
> + if (virStreamFinish(data->st) < 0)
> + virReportError(errno, "%s",
> + _("tunnelled migration failed to finish stream"));
AFAICT virStreamFinish also reports error.
Besides the nits, looks good. I'll find some time tomorrow to test the patches.
Thanks!
Regards,
Jim
> + cleanup:
> + VIR_FREE(buffer);
> +
> + return;
> +}
> +
> +struct libxlTunnelControl {
> + libxlTunnelMigrationThread tmThread;
> + virThread thread;
> + int dataFD[2];
> +};
> +
> +static int
> +libxlMigrationStartTunnel(libxlDriverPrivatePtr driver,
> + virDomainObjPtr vm,
> + unsigned long flags,
> + virStreamPtr st,
> + struct libxlTunnelControl *tc)
> +{
> + libxlTunnelMigrationThread *arg = NULL;
> + int ret = -1;
> +
> + if (VIR_ALLOC(tc) < 0)
> + goto out;
> +
> + tc->dataFD[0] = -1;
> + tc->dataFD[1] = -1;
> + if (pipe(tc->dataFD) < 0) {
> + virReportError(errno, "%s", _("Unable to make pipes"));
> + goto out;
> + }
> +
> + arg = &tc->tmThread;
> + /* Read from pipe */
> + arg->srcFD = tc->dataFD[0];
> + /* Write to dest stream */
> + arg->st = st;
> + if (virThreadCreate(&tc->thread, true,
> + libxlTunnel3MigrationFunc, arg) < 0) {
> + virReportError(errno, "%s",
> + _("Unable to create tunnel migration thread"));
> + goto out;
> + }
> +
> + virObjectUnlock(vm);
> + /* Send data to pipe */
> + ret = libxlDoMigrateSend(driver, vm, flags, tc->dataFD[1]);
> + virObjectLock(vm);
> +
> + out:
> + /* libxlMigrationStopTunnel will be called in libxlDoMigrateP2P to free
> + * all resources for us. */
> + return ret;
> +}
> +
> +static void libxlMigrationStopTunnel(struct libxlTunnelControl *tc)
> +{
> + if (!tc)
> + return;
> +
> + virThreadCancel(&tc->thread);
> + virThreadJoin(&tc->thread);
> +
> + VIR_FORCE_CLOSE(tc->dataFD[0]);
> + VIR_FORCE_CLOSE(tc->dataFD[1]);
> + VIR_FREE(tc);
> +}
> +
> +/* This function is a simplification of virDomainMigrateVersion3Full and
> + * restricting it to migration v3 with params since it was the first to be
> + * introduced in libxl.
> */
> static int
> libxlDoMigrateP2P(libxlDriverPrivatePtr driver,
> @@ -761,6 +990,9 @@ libxlDoMigrateP2P(libxlDriverPrivatePtr driver,
> bool cancelled = true;
> virErrorPtr orig_err = NULL;
> int ret = -1;
> + /* For tunnel migration */
> + virStreamPtr st = NULL;
> + struct libxlTunnelControl *tc = NULL;
>
> dom_xml = libxlDomainMigrationBegin(sconn, vm, xmlin,
> &cookieout, &cookieoutlen);
> @@ -788,29 +1020,40 @@ libxlDoMigrateP2P(libxlDriverPrivatePtr driver,
>
> VIR_DEBUG("Prepare3");
> virObjectUnlock(vm);
> - ret = dconn->driver->domainMigratePrepare3Params
> - (dconn, params, nparams, cookieout, cookieoutlen, NULL, NULL, &uri_out, destflags);
> + if (flags & VIR_MIGRATE_TUNNELLED) {
> + if (!(st = virStreamNew(dconn, 0)))
> + goto cleanup;
> + ret = dconn->driver->domainMigratePrepareTunnel3Params
> + (dconn, st, params, nparams, cookieout, cookieoutlen, NULL, NULL, destflags);
> + } else {
> + ret = dconn->driver->domainMigratePrepare3Params
> + (dconn, params, nparams, cookieout, cookieoutlen, NULL, NULL, &uri_out, destflags);
> + }
> virObjectLock(vm);
>
> if (ret == -1)
> goto cleanup;
>
> - if (uri_out) {
> - if (virTypedParamsReplaceString(¶ms, &nparams,
> - VIR_MIGRATE_PARAM_URI, uri_out) < 0) {
> - orig_err = virSaveLastError();
> + if (!(flags & VIR_MIGRATE_TUNNELLED)) {
> + if (uri_out) {
> + if (virTypedParamsReplaceString(¶ms, &nparams,
> + VIR_MIGRATE_PARAM_URI, uri_out) < 0) {
> + orig_err = virSaveLastError();
> + goto finish;
> + }
> + } else {
> + virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> + _("domainMigratePrepare3 did not set uri"));
> goto finish;
> }
> - } else {
> - virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> - _("domainMigratePrepare3 did not set uri"));
> - goto finish;
> }
>
> VIR_DEBUG("Perform3 uri=%s", NULLSTR(uri_out));
> - ret = libxlDomainMigrationPerform(driver, vm, NULL, NULL,
> - uri_out, NULL, flags);
> -
> + if (flags & VIR_MIGRATE_TUNNELLED)
> + ret = libxlMigrationStartTunnel(driver, vm, flags, st, tc);
> + else
> + ret = libxlDomainMigrationPerform(driver, vm, NULL, NULL,
> + uri_out, NULL, flags);
> if (ret < 0)
> orig_err = virSaveLastError();
>
> @@ -848,6 +1091,11 @@ libxlDoMigrateP2P(libxlDriverPrivatePtr driver,
> vm->def->name);
>
> cleanup:
> + if (flags & VIR_MIGRATE_TUNNELLED) {
> + libxlMigrationStopTunnel(tc);
> + virObjectUnref(st);
> + }
> +
> if (ddomain) {
> virObjectUnref(ddomain);
> ret = 0;
> diff --git a/src/libxl/libxl_migration.h b/src/libxl/libxl_migration.h
> index 8a074a0..fcea558 100644
> --- a/src/libxl/libxl_migration.h
> +++ b/src/libxl/libxl_migration.h
> @@ -29,6 +29,7 @@
> # define LIBXL_MIGRATION_FLAGS \
> (VIR_MIGRATE_LIVE | \
> VIR_MIGRATE_PEER2PEER | \
> + VIR_MIGRATE_TUNNELLED | \
> VIR_MIGRATE_PERSIST_DEST | \
> VIR_MIGRATE_UNDEFINE_SOURCE | \
> VIR_MIGRATE_PAUSED)
> @@ -53,6 +54,14 @@ libxlDomainMigrationPrepareDef(libxlDriverPrivatePtr driver,
> const char *dname);
>
> int
> +libxlDomainMigrationPrepareTunnel3(virConnectPtr dconn,
> + virStreamPtr st,
> + virDomainDefPtr *def,
> + const char *cookiein,
> + int cookieinlen,
> + unsigned int flags);
> +
> +int
> libxlDomainMigrationPrepare(virConnectPtr dconn,
> virDomainDefPtr *def,
> const char *uri_in,
>
More information about the libvir-list
mailing list