[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[libvirt] [PATCH 11/11] Add a plugin for the 'sanlock' project



Sanlock is a project that implements a disk-paxos locking
algorithm. This is suitable for cluster deployments with
shared storage.

* src/Makefile.am: Add dlopen plugin for sanlock
* src/locking/lock_driver_sanlock.c: Sanlock driver
---
 po/POTFILES.in                    |    1 +
 src/Makefile.am                   |   12 +
 src/libvirt_private.syms          |    1 +
 src/locking/lock_driver_sanlock.c |  452 +++++++++++++++++++++++++++++++++++++
 4 files changed, 466 insertions(+), 0 deletions(-)
 create mode 100644 src/locking/lock_driver_sanlock.c

diff --git a/po/POTFILES.in b/po/POTFILES.in
index 47f2f20..302b9c0 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -30,6 +30,7 @@ src/interface/netcf_driver.c
 src/internal.h
 src/libvirt.c
 src/locking/lock_manager.c
+src/locking/lock_driver_sanlock.c
 src/lxc/lxc_container.c
 src/lxc/lxc_conf.c
 src/lxc/lxc_controller.c
diff --git a/src/Makefile.am b/src/Makefile.am
index b68a9b4..f56ff17 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -97,6 +97,9 @@ DRIVER_SOURCES =						\
 		locking/lock_driver_nop.h locking/lock_driver_nop.c \
 		locking/domain_lock.h locking/domain_lock.c
 
+LOCK_DRIVER_SANLOCK_SOURCES = \
+		locking/lock_driver_sanlock.c
+
 
 # XML configuration format handling sources
 # Domain driver generic impl APIs
@@ -1148,6 +1151,15 @@ libvirt_qemu_la_CFLAGS = $(AM_CFLAGS)
 libvirt_qemu_la_LIBADD = libvirt.la $(CYGWIN_EXTRA_LIBADD)
 EXTRA_DIST += $(LIBVIRT_QEMU_SYMBOL_FILE)
 
+
+lockdriverdir = $(libdir)/libvirt/lock-driver
+lockdriver_LTLIBRARIES = sanlock.la
+
+sanlock_la_SOURCES = $(LOCK_DRIVER_SANLOCK_SOURCES)
+sanlock_la_CFLAGS = $(AM_CLFAGS)
+sanlock_la_LDFLAGS = -no-version -module
+sanlock_la_LIBADD = -lsanlock
+
 libexec_PROGRAMS =
 
 if WITH_STORAGE_DISK
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 8005f20..b2f11b8 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -585,6 +585,7 @@ virVMOperationTypeToString;
 # memory.h
 virAlloc;
 virAllocN;
+virAllocVar;
 virExpandN;
 virFree;
 virReallocN;
diff --git a/src/locking/lock_driver_sanlock.c b/src/locking/lock_driver_sanlock.c
new file mode 100644
index 0000000..90afe18
--- /dev/null
+++ b/src/locking/lock_driver_sanlock.c
@@ -0,0 +1,452 @@
+/*
+ * lock_driver_sanlock.c: A lock driver for Sanlock
+ *
+ * Copyright (C) 2010-2011 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+ *
+ */
+
+#include <config.h>
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/types.h>
+
+#include <sanlock.h>
+#include <sanlock_resource.h>
+
+#include "lock_driver.h"
+#include "logging.h"
+#include "virterror_internal.h"
+#include "memory.h"
+#include "util.h"
+#include "files.h"
+
+#define VIR_FROM_THIS VIR_FROM_LOCKING
+
+#define virLockError(code, ...)                                         \
+    virReportErrorHelper(NULL, VIR_FROM_THIS, code, __FILE__,           \
+                             __FUNCTION__, __LINE__, __VA_ARGS__)
+
+struct snlk_con {
+    char vm_name[SANLK_NAME_LEN];
+    char vm_uuid[VIR_UUID_BUFLEN];
+    unsigned int vm_id;
+    unsigned int vm_pid;
+    unsigned int flags;
+    int sock;
+    int res_count;
+    struct sanlk_resource *res_args[SANLK_MAX_RESOURCES];
+};
+
+/*
+ * sanlock plugin for the libvirt virLockManager API
+ */
+
+static int drv_snlk_init(unsigned int version ATTRIBUTE_UNUSED,
+                         unsigned int flags)
+{
+    virCheckFlags(VIR_LOCK_MANAGER_MODE_CONTENT, -1);
+    return 0;
+}
+
+static int drv_snlk_deinit(void)
+{
+    virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                 _("Unloading sanlock plugin is forbidden"));
+    return -1;
+}
+
+static int drv_snlk_new(virLockManagerPtr man,
+                        unsigned int type,
+                        size_t nparams,
+                        virLockManagerParamPtr params,
+                        unsigned int flags)
+{
+    virLockManagerParamPtr param;
+    struct snlk_con *con;
+    int i;
+
+    virCheckFlags(VIR_LOCK_MANAGER_MODE_CONTENT, -1);
+
+    if (type != VIR_LOCK_MANAGER_OBJECT_TYPE_DOMAIN) {
+        virLockError(VIR_ERR_INTERNAL_ERROR,
+                     _("Unsupported object type %d"), type);
+        return -1;
+    }
+
+    if (VIR_ALLOC(con) < 0) {
+        virReportOOMError();
+        return -1;
+    }
+
+    con->flags = flags;
+    con->sock = -1;
+
+    for (i = 0; i < nparams; i++) {
+        param = &params[i];
+
+        if (STREQ(param->key, "uuid")) {
+            memcpy(con->vm_uuid, param->value.uuid, 16);
+        } else if (STREQ(param->key, "name")) {
+            if (!virStrcpy(con->vm_name, param->value.str, SANLK_NAME_LEN)) {
+                virLockError(VIR_ERR_INTERNAL_ERROR,
+                             _("Domain name '%s' exceeded %d characters"),
+                             param->value.str, SANLK_NAME_LEN);
+                goto error;
+            }
+        } else if (STREQ(param->key, "pid")) {
+            con->vm_pid = param->value.ui;
+        } else if (STREQ(param->key, "id")) {
+            con->vm_id = param->value.ui;
+        }
+    }
+
+    man->privateData = con;
+    return 0;
+
+error:
+    VIR_FREE(con);
+    return -1;
+}
+
+static void drv_snlk_free(virLockManagerPtr man)
+{
+    struct snlk_con *con = man->privateData;
+
+    DEBUG("man=%p sock=%d", man, con->sock);
+#if 0
+    /* We do *not* want to close the socket here. We need the
+     * socket to keep alive other sanlock will fence the
+     * process. The socket will be explicitly closed before
+     * free, in the  release_object method, if neccessary.
+     */
+    VIR_FORCE_CLOSE(con->sock);
+#endif
+    VIR_FREE(con);
+    man->privateData = NULL;
+}
+
+static int add_con_resource(struct snlk_con *con,
+                            const char *name,
+                            size_t nparams,
+                            virLockManagerParamPtr params)
+{
+    virLockManagerParamPtr param;
+    struct sanlk_resource *res;
+    int i;
+
+    if (VIR_ALLOC_VAR(res, struct sanlk_disk, 1) < 0) {
+        virReportOOMError();
+        return -1;
+    }
+
+    res->num_disks = 1;
+    if (!virStrcpy(res->name, name, SANLK_NAME_LEN)) {
+        virLockError(VIR_ERR_INTERNAL_ERROR,
+                     _("Resource name '%s' exceeds %d characters"),
+                     name, SANLK_NAME_LEN);
+        goto error;
+    }
+
+    for (i = 0; i < nparams; i++) {
+        param = &params[i];
+
+        if (STREQ(param->key, "path")) {
+            if (!virStrcpy(res->disks[0].path, param->value.str, SANLK_PATH_LEN)) {
+                virLockError(VIR_ERR_INTERNAL_ERROR,
+                             _("Lease path '%s' exceeds %d characters"),
+                             param->value.str, SANLK_PATH_LEN);
+                goto error;
+            }
+        } else if (STREQ(param->key, "offset")) {
+            res->disks[0].offset = param->value.ul;
+        }
+    }
+
+    con->res_args[con->res_count] = res;
+    con->res_count++;
+    return 0;
+
+error:
+    VIR_FREE(res);
+    return -1;
+}
+
+static int drv_snlk_add_resource(virLockManagerPtr man,
+                                 unsigned int type,
+                                 const char *name,
+                                 size_t nparams,
+                                 virLockManagerParamPtr params,
+                                 unsigned int flags ATTRIBUTE_UNUSED)
+{
+    struct snlk_con *con = man->privateData;
+    /* must be called before acquire_object */
+    if (con->sock != -1) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Cannot add resources to an existing lock"));
+        return -1;
+    }
+
+    if (con->res_count == SANLK_MAX_RESOURCES) {
+        virLockError(VIR_ERR_INTERNAL_ERROR,
+                     _("Too many resources %d for object"),
+                     SANLK_MAX_RESOURCES);
+        return -1;
+    }
+
+    if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE)
+        return 0;
+
+    if (add_con_resource(con, name, nparams, params) < 0)
+        return -1;
+
+    return 0;
+}
+
+static int drv_snlk_acquire_object(virLockManagerPtr man,
+                                   const char *state,
+                                   unsigned int flags ATTRIBUTE_UNUSED)
+{
+    struct snlk_con *con = man->privateData;
+    struct sanlk_options *opt = NULL;
+    int i, rv, sock;
+    int pid = getpid();
+
+    /* acquire_object can be called only once */
+    if (con->sock != -1) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Object lock is already held"));
+        return -1;
+    }
+
+    if (con->vm_pid != pid) {
+        virLockError(VIR_ERR_INTERNAL_ERROR,
+                     _("Object lock attempt from pid %d, expected %d"),
+                     pid, con->vm_pid);
+        return -1;
+    }
+
+    if (VIR_ALLOC_VAR(opt, char, state ? strlen(state) : 0) < 0) {
+        virReportOOMError();
+        return -1;
+    }
+
+    if (!virStrcpy(opt->owner_name, con->vm_name, SANLK_NAME_LEN)) {
+        virLockError(VIR_ERR_INTERNAL_ERROR,
+                     _("Domain name '%s' exceeded %d characters"),
+                     con->vm_name, SANLK_NAME_LEN);
+        goto error;
+    }
+
+    if (state) {
+        opt->flags = SANLK_FLG_INCOMING;
+        opt->len = strlen(state);
+        strcpy(opt->str, state);
+    }
+
+    VIR_DEBUG0("Register sanlock");
+    sock = sanlock_register();
+    if (sock < 0) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Failed to open socket to sanlock daemon"));
+        goto error;
+    }
+    VIR_DEBUG("Acquiring object %u", con->res_count);
+    rv = sanlock_acquire(sock, -1, con->res_count, con->res_args, opt);
+    VIR_DEBUG("Acquire result %d", rv);
+    if (rv < 0) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Failed to acquire lock"));
+        goto error;
+    }
+    VIR_FREE(opt);
+
+    for (i = 0; i < con->res_count; i++)
+        VIR_FREE(con->res_args[i]);
+
+    con->sock = sock;
+
+    return 0;
+
+error:
+    VIR_FORCE_CLOSE(sock);
+    VIR_FREE(opt);
+    return -1;
+}
+
+static int drv_snlk_attach_object(virLockManagerPtr man ATTRIBUTE_UNUSED,
+                                  unsigned int flags ATTRIBUTE_UNUSED)
+{
+    return 0;
+}
+
+static int drv_snlk_detach_object(virLockManagerPtr man ATTRIBUTE_UNUSED,
+                                  unsigned int flags ATTRIBUTE_UNUSED)
+{
+    return 0;
+}
+
+static int drv_snlk_release_object(virLockManagerPtr man ATTRIBUTE_UNUSED,
+                                   unsigned int flags ATTRIBUTE_UNUSED)
+{
+    struct snlk_con *con = man->privateData;
+
+    if (con->sock == -1) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Cannot release object that is not locked"));
+        return -1;
+    }
+    VIR_FORCE_CLOSE(con->sock);
+    return 0;
+}
+
+static int drv_snlk_get_state(virLockManagerPtr man ATTRIBUTE_UNUSED,
+                              char **state,
+                              unsigned int flags ATTRIBUTE_UNUSED)
+{
+    *state = NULL;
+
+    return 0;
+}
+
+
+static int drv_snlk_acquire_resource(virLockManagerPtr man,
+                                     unsigned int type,
+                                     const char *name,
+                                     size_t nparams,
+                                     virLockManagerParamPtr params,
+                                     unsigned int flags ATTRIBUTE_UNUSED)
+{
+    struct snlk_con *con = man->privateData;
+    struct sanlk_options opt;
+    int rv;
+
+    if (con->sock != -1) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Cannot acquire resource on unlocked object"));
+        return -1;
+    }
+
+    if (!con->vm_pid) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Cannot acquire resource on unlocked object"));
+        return -1;
+    }
+
+    if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE)
+        return 0;
+
+    if (add_con_resource(con, name, nparams, params) < 0)
+        return -1;
+
+    /* Setting REACQUIRE tells sanlock that if con->vm_pid previously held
+       and released the resource, we need to ensure no other host has
+       acquired a lease on it in the mean time.  If this is a new resource
+       that the pid hasn't held before, then REACQUIRE will have no effect
+       since sanlock will have no memory of a previous version. */
+
+    memset(&opt, 0, sizeof(struct sanlk_options));
+    if (!virStrcpy(opt.owner_name, con->vm_name, SANLK_NAME_LEN)) {
+        virLockError(VIR_ERR_INTERNAL_ERROR,
+                     _("Domain name '%s' exceeds %d characters"),
+                     con->vm_name, SANLK_NAME_LEN);
+        return -1;
+    }
+    opt.flags = SANLK_FLG_REACQUIRE;
+    opt.len = 0;
+
+    rv = sanlock_acquire(-1, con->vm_pid, con->res_count, con->res_args, &opt);
+
+    VIR_FREE(con->res_args[0]);
+
+    if (rv < 0) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Unable to acquire resource"));
+        return -1;
+    }
+
+    return 0;
+}
+
+static int drv_snlk_release_resource(virLockManagerPtr man,
+                                     unsigned int type,
+                                     const char *name,
+                                     size_t nparams,
+                                     virLockManagerParamPtr params,
+                                     unsigned int flags ATTRIBUTE_UNUSED)
+{
+    struct snlk_con *con = man->privateData;
+    int rv;
+
+    if (con->sock != -1) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Cannot acquire resource on unlocked object"));
+        return -1;
+    }
+
+    if (!con->vm_pid) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Cannot acquire resource on unlocked object"));
+        return -1;
+    }
+
+    if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE)
+        return 0;
+
+    if (add_con_resource(con, name, nparams, params) < 0)
+        return -1;
+
+    rv = sanlock_release(-1, con->vm_pid, con->res_count, con->res_args);
+
+    VIR_FREE(con->res_args[0]);
+
+    if (rv < 0) {
+        virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Unable to release resource"));
+        return -1;
+    }
+
+    return 0;
+}
+
+virLockDriver virLockDriverImpl =
+{
+    .version = VIR_LOCK_MANAGER_VERSION,
+    .flags = VIR_LOCK_MANAGER_MODE_CONTENT,
+
+    .drvInit = drv_snlk_init,
+    .drvDeinit = drv_snlk_deinit,
+
+    .drvNew = drv_snlk_new,
+    .drvFree = drv_snlk_free,
+
+    .drvAddResource = drv_snlk_add_resource,
+
+    .drvAcquireObject = drv_snlk_acquire_object,
+    .drvAttachObject = drv_snlk_attach_object,
+    .drvDetachObject = drv_snlk_detach_object,
+    .drvReleaseObject = drv_snlk_release_object,
+
+    .drvGetState = drv_snlk_get_state,
+
+    .drvAcquireResource = drv_snlk_acquire_resource,
+    .drvReleaseResource = drv_snlk_release_resource,
+};
-- 
1.7.3.4


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]