[libvirt] [PATCH 1/3] Add a plugin dlm for lock manager
river
lfu at suse.com
Mon Feb 5 09:07:54 UTC 2018
dlm is implemented by linux kernel, provides userspace API by
'libdlm' to lock/unlock resource, cooperated with cluster
communication software, such as corosync, could satisfy the
demands of libvirt lock manager.
Signed-off-by: river <lfu at suse.com>
---
configure.ac | 6 +
m4/virt-cpg.m4 | 37 ++
m4/virt-dlm.m4 | 36 ++
src/Makefile.am | 15 +
src/locking/lock_driver_dlm.c | 1056 +++++++++++++++++++++++++++++++++++++++++
src/util/virlist.h | 110 +++++
6 files changed, 1260 insertions(+)
create mode 100644 m4/virt-cpg.m4
create mode 100644 m4/virt-dlm.m4
create mode 100644 src/locking/lock_driver_dlm.c
create mode 100644 src/util/virlist.h
diff --git a/configure.ac b/configure.ac
index 4cccf7f4d..4ad90470d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -265,6 +265,8 @@ LIBVIRT_ARG_PM_UTILS
LIBVIRT_ARG_POLKIT
LIBVIRT_ARG_READLINE
LIBVIRT_ARG_SANLOCK
+LIBVIRT_ARG_CPG
+LIBVIRT_ARG_DLM
LIBVIRT_ARG_SASL
LIBVIRT_ARG_SELINUX
LIBVIRT_ARG_SSH2
@@ -307,6 +309,8 @@ LIBVIRT_CHECK_POLKIT
LIBVIRT_CHECK_PTHREAD
LIBVIRT_CHECK_READLINE
LIBVIRT_CHECK_SANLOCK
+LIBVIRT_CHECK_CPG
+LIBVIRT_CHECK_DLM
LIBVIRT_CHECK_SASL
LIBVIRT_CHECK_SELINUX
LIBVIRT_CHECK_SSH2
@@ -1005,6 +1009,8 @@ LIBVIRT_RESULT_POLKIT
LIBVIRT_RESULT_RBD
LIBVIRT_RESULT_READLINE
LIBVIRT_RESULT_SANLOCK
+LIBVIRT_RESULT_CPG
+LIBVIRT_RESULT_DLM
LIBVIRT_RESULT_SASL
LIBVIRT_RESULT_SELINUX
LIBVIRT_RESULT_SSH2
diff --git a/m4/virt-cpg.m4 b/m4/virt-cpg.m4
new file mode 100644
index 000000000..27acda665
--- /dev/null
+++ b/m4/virt-cpg.m4
@@ -0,0 +1,37 @@
+dnl The libcpg.so library
+dnl
+dnl Copyright (C) 2018 SUSE LINUX Products, Beijing, China.
+dnl
+dnl This library is free software; you can redistribute it and/or
+dnl modify it under the terms of the GNU Lesser General Public
+dnl License as published by the Free Software Foundation; either
+dnl version 2.1 of the License, or (at your option) any later version.
+dnl
+dnl This library is distributed in the hope that it will be useful,
+dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
+dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+dnl Lesser General Public License for more details.
+dnl
+dnl You should have received a copy of the GNU Lesser General Public
+dnl License along with this library. If not, see
+dnl <http://www.gnu.org/licenses/>.
+dnl
+
+AC_DEFUN([LIBVIRT_ARG_CPG],[
+ LIBVIRT_ARG_WITH_FEATURE([CPG], [cluster engine CPG library], [check])
+])
+
+AC_DEFUN([LIBVIRT_CHECK_CPG],[
+ dnl in some distribution, Version is `UNKNOW` in libcpg.pc
+ if test "x$with_cpg" != "xno"; then
+ PKG_CHECK_MODULES([CPG], [libcpg], [
+ with_cpg=yes
+ ],[
+ with_cpg=no
+ ])
+ fi
+])
+
+AC_DEFUN([LIBVIRT_RESULT_CPG],[
+ LIBVIRT_RESULT_LIB([CPG])
+])
diff --git a/m4/virt-dlm.m4 b/m4/virt-dlm.m4
new file mode 100644
index 000000000..dc5f5f152
--- /dev/null
+++ b/m4/virt-dlm.m4
@@ -0,0 +1,36 @@
+dnl The libdlm.so library
+dnl
+dnl Copyright (C) 2018 SUSE LINUX Products, Beijing, China.
+dnl
+dnl This library is free software; you can redistribute it and/or
+dnl modify it under the terms of the GNU Lesser General Public
+dnl License as published by the Free Software Foundation; either
+dnl version 2.1 of the License, or (at your option) any later version.
+dnl
+dnl This library is distributed in the hope that it will be useful,
+dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
+dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+dnl Lesser General Public License for more details.
+dnl
+dnl You should have received a copy of the GNU Lesser General Public
+dnl License along with this library. If not, see
+dnl <http://www.gnu.org/licenses/>.
+dnl
+
+AC_DEFUN([LIBVIRT_ARG_DLM],[
+ LIBVIRT_ARG_WITH_FEATURE([DLM], [Distributed Lock Manager library], [check])
+])
+
+AC_DEFUN([LIBVIRT_CHECK_DLM],[
+ AC_REQUIRE([LIBVIRT_CHECK_CPG])
+ LIBVIRT_CHECK_PKG([DLM], [libdlm], [4.0.0])
+
+ if test "x$with_dlm" == "xyes" && test "x$with_cpg" != "xyes"; then
+ AC_MSG_ERROR([You must install libcpg to build dlm lock])
+ fi
+])
+
+AC_DEFUN([LIBVIRT_RESULT_DLM],[
+ AC_REQUIRE([LIBVIRT_RESULT_CPG])
+ LIBVIRT_RESULT_LIB([DLM])
+])
diff --git a/src/Makefile.am b/src/Makefile.am
index 79adc9ba5..8742921fa 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -305,6 +305,10 @@ DRIVER_SOURCES = \
logging/log_manager.c logging/log_manager.h \
$(NULL)
+LOCK_DRIVER_DLM_SOURCES = \
+ locking/lock_driver_dlm.c \
+ $(NULL)
+
LOCK_DRIVER_SANLOCK_SOURCES = \
locking/lock_driver_sanlock.c
@@ -2879,6 +2883,17 @@ virtlogd.socket: logging/virtlogd.socket.in $(top_builddir)/config.status
< $< > $@-t && \
mv $@-t $@
+if WITH_DLM
+lockdriver_LTLIBRARIES += dlm.la
+dlm_la_SOURCES = $(LOCK_DRIVER_DLM_SOURCES)
+dlm_la_CFLAGS = -I$(srcdir)/conf $(AM_CFLAGS)
+dlm_la_LDFLAGS = -module -avoid-version $(AM_LDFLAGS)
+dlm_la_LIBADD = ../gnulib/lib/libgnu.la \
+ $(CPG_LIBS) \
+ $(DLM_LIBS)
+else ! WITH_DLM
+EXTRA_DIST += $(LOCK_DRIVER_DLM_SOURCES)
+endif
if WITH_SANLOCK
lockdriver_LTLIBRARIES += sanlock.la
diff --git a/src/locking/lock_driver_dlm.c b/src/locking/lock_driver_dlm.c
new file mode 100644
index 000000000..e197c0bdf
--- /dev/null
+++ b/src/locking/lock_driver_dlm.c
@@ -0,0 +1,1056 @@
+/*
+ * lock_driver_dlm.c: a lock driver for dlm
+ *
+ * Copyright (C) 2018 SUSE LINUX Products, Beijing, China.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <config.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <stdint.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+
+#include <corosync/cpg.h>
+#include <libdlm.h>
+
+#include "lock_driver.h"
+#include "viralloc.h"
+#include "virconf.h"
+#include "vircrypto.h"
+#include "virerror.h"
+#include "virfile.h"
+#include "virlist.h"
+#include "virlog.h"
+#include "virstring.h"
+#include "virthread.h"
+#include "viruuid.h"
+
+#define VIR_FROM_THIS VIR_FROM_LOCKING
+
+#define DLM_LOCKSPACE_MODE 0600
+#define DLM_LOCKSPACE_NAME "libvirt"
+
+#define LOCK_RECORD_FILE_MODE 0644
+#define LOCK_RECORD_FILE_PATH "/tmp/libvirtd-dlm-file"
+
+#define PRMODE "PRMODE"
+#define EXMODE "EXMODE"
+
+#define STATUS "STATUS"
+#define RESOURCE_NAME "RESOURCE_NAME"
+#define LOCK_ID "LOCK_ID"
+#define LOCK_MODE "LOCK_MODE"
+#define VM_PID "VM_PID"
+
+#define BUFFERLEN 128
+
+/* This will be set after dlm_controld is started. */
+#define DLM_CLUSTER_NAME_PATH "/sys/kernel/config/dlm/cluster/cluster_name"
+
+VIR_LOG_INIT("locking.lock_driver_dlm");
+
+typedef struct _virLockInformation virLockInformation;
+typedef virLockInformation *virLockInformationPtr;
+
+typedef struct _virLockManagerDlmResource virLockManagerDlmResource;
+typedef virLockManagerDlmResource *virLockManagerDlmResourcePtr;
+
+typedef struct _virLockManagerDlmPrivate virLockManagerDlmPrivate;
+typedef virLockManagerDlmPrivate *virLockManagerDlmPrivatePtr;
+
+typedef struct _virLockManagerDlmDriver virLockManagerDlmDriver;
+typedef virLockManagerDlmDriver *virLockManagerDlmDriverPtr;
+
+typedef struct _virListWait virListWait;
+typedef virListWait *virListWaitPtr;
+
+struct _virLockInformation {
+ virListHead entry;
+ char *name;
+ uint32_t mode;
+ uint32_t lkid;
+ pid_t vm_pid;
+};
+
+struct _virLockManagerDlmResource {
+ char *name;
+ uint32_t mode;
+};
+
+struct _virLockManagerDlmPrivate {
+ unsigned char vm_uuid[VIR_UUID_BUFLEN];
+ char *vm_name;
+ pid_t vm_pid;
+ int vm_id;
+
+ size_t nresources;
+ virLockManagerDlmResourcePtr resources;
+
+ bool hasRWDisks;
+};
+
+struct _virLockManagerDlmDriver {
+ bool autoDiskLease;
+ bool requireLeaseForDisks;
+
+ bool purgeLockspace;
+ char *lockspaceName;
+ char *lockRecordFilePath;
+};
+
+struct _virListWait {
+ virMutex listMutex;
+ virMutex fileMutex;
+ virListHead list;
+};
+
+static virLockManagerDlmDriverPtr driver;
+static dlm_lshandle_t lockspace;
+static virListWait lockListWait;
+
+static int virLockManagerDlmLoadConfig(const char *configFile)
+{
+ virConfPtr conf = NULL;
+ int ret = -1;
+
+ if (access(configFile, R_OK) == -1) {
+ if (errno != ENOENT) {
+ virReportSystemError(errno,
+ _("Unable to access config file %s"),
+ configFile);
+ return -1;
+ }
+ return 0;
+ }
+
+ if (!(conf = virConfReadFile(configFile, 0)))
+ return -1;
+
+ if (virConfGetValueBool(conf, "auto_disk_leases", &driver->autoDiskLease) < 0)
+ goto cleanup;
+
+ driver->requireLeaseForDisks = !driver->autoDiskLease;
+ if (virConfGetValueBool(conf, "require_lease_for_disks", &driver->requireLeaseForDisks) < 0)
+ goto cleanup;
+
+ if (virConfGetValueBool(conf, "purge_lockspace", &driver->purgeLockspace) < 0)
+ goto cleanup;
+
+ if (virConfGetValueString(conf, "lockspace_name", &driver->lockspaceName) < 0)
+ goto cleanup;
+
+ if (virConfGetValueString(conf, "lock_record_file_path", &driver->lockRecordFilePath) < 0)
+ goto cleanup;
+
+ ret = 0;
+
+ cleanup:
+ virConfFree(conf);
+ return ret;
+}
+
+static int virLockManagerDlmToModeUint(const char *token)
+{
+ if (STREQ(token, PRMODE))
+ return LKM_PRMODE;
+ if (STREQ(token, EXMODE))
+ return LKM_EXMODE;
+
+ return 0;
+}
+
+static const char *virLockManagerDlmToModeText(const uint32_t mode)
+{
+ switch (mode) {
+ case LKM_PRMODE:
+ return PRMODE;
+ case LKM_EXMODE:
+ return EXMODE;
+ default:
+ return NULL;
+ }
+}
+
+static virLockInformationPtr virLockManagerDlmRecordLock(const char *name,
+ const uint32_t mode,
+ const uint32_t lkid,
+ const pid_t vm_pid)
+{
+ virLockInformationPtr lock = NULL;
+
+ if (VIR_ALLOC(lock) < 0)
+ goto error;
+
+ if (VIR_STRDUP(lock->name, name) < 0)
+ goto error;
+
+ lock->mode = mode;
+ lock->lkid = lkid;
+ lock->vm_pid = vm_pid;
+
+ virMutexLock(&(lockListWait.listMutex));
+ virListAddTail(&lock->entry, &(lockListWait.list));
+ virMutexUnlock(&(lockListWait.listMutex));
+
+ VIR_DEBUG("record lock sucessfully, lockName=%s lockMode=%s lockId=%d",
+ NULLSTR(name), NULLSTR(virLockManagerDlmToModeText(mode)), lkid);
+
+ return lock;
+
+ error:
+ if (lock)
+ VIR_FREE(lock->name);
+ VIR_FREE(lock);
+ return NULL;
+}
+
+static void virLockManagerDlmWriteLock(virLockInformationPtr lock, int fd, bool status)
+{
+ char buffer[BUFFERLEN] = {0};
+ off_t offset = 0, rv = 0;
+
+ if (!lock) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("lock is NULL"));
+ return;
+ }
+
+ /*
+ * STATUS RESOURCE_NAME LOCK_MODE VM_PID\n
+ * 6 64 9 10
+ * 93 = 6 + 1 + 64 + 1 + 9 + 1 + 10 + 1
+ */
+ offset = 93 * lock->lkid;
+ rv = lseek(fd, offset, SEEK_SET);
+ if (rv < 0) {
+ virReportSystemError(errno,
+ _("unable to lseek fd '%d'"),
+ fd);
+ return;
+ }
+
+ snprintf(buffer, sizeof(buffer), "%6d %64s %9s %10jd\n", \
+ status, lock->name,
+ NULLSTR(virLockManagerDlmToModeText(lock->mode)),
+ (intmax_t)lock->vm_pid);
+
+ if (safewrite(fd, buffer, strlen(buffer)) != strlen(buffer)) {
+ virReportSystemError(errno,
+ _("unable to write lock information '%s' to file '%s'"),
+ buffer, NULLSTR(driver->lockRecordFilePath));
+ return;
+ }
+
+ VIR_DEBUG("write '%s' to fd=%d", buffer, fd);
+
+ fdatasync(fd);
+
+ return;
+}
+
+static void virLockManagerDlmAdoptLock(char *raw) {
+ char *str = NULL, *subtoken = NULL, *saveptr = NULL, *endptr = NULL;
+ int i = 0, status = 0;
+ char *name = NULL;
+ uint32_t mode = 0;
+ pid_t vm_pid = 0;
+ struct dlm_lksb lksb = {0};
+
+ /* every line is the following format:
+ * STATUS RESOURCE_NAME LOCK_MODE VM_PID
+ */
+ for (i = 0, str = raw, status = 0; ; str = NULL, i++) {
+ subtoken = strtok_r(str, " \n", &saveptr);
+ if (subtoken == NULL)
+ break;
+
+ switch(i) {
+ case 0:
+ if (virStrToLong_i(subtoken, &endptr, 10, &status) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("cannot extract lock status '%s'"), subtoken);
+ goto cleanup;
+ }
+ break;
+ case 1:
+ if (VIR_STRDUP(name, subtoken) != 1)
+ goto cleanup;
+ break;
+ case 2:
+ mode = virLockManagerDlmToModeUint(subtoken);
+ if (!mode)
+ goto cleanup;
+ break;
+ case 3:
+ if ((virStrToLong_i(subtoken, &endptr, 10, &vm_pid) < 0) || !vm_pid) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("cannot extract lock vm_pid '%s'"), subtoken);
+ goto cleanup;
+ }
+ break;
+ default:
+ goto cleanup;
+ break;
+ }
+
+ if (status != 1)
+ goto cleanup;
+ }
+
+ if (i != 4)
+ goto cleanup;
+
+ /* copy from `lm_adopt_dlm` in daemons/lvmlockd/lvmlockd-dlm.c of lvm2:
+ * dlm returns 0 for success, -EAGAIN if an orphan is
+ * found with another mode, and -ENOENT if no orphan.
+ *
+ * cast/bast/param are (void *)1 because the kernel
+ * returns errors if some are null.
+ */
+
+ status = dlm_ls_lockx(lockspace, mode, &lksb, LKF_PERSISTENT|LKF_ORPHAN,
+ name, strlen(name), 0,
+ (void *)1, (void *)1, (void *)1,
+ NULL, NULL);
+ if (status) {
+ virReportSystemError(errno,
+ _("unable to adopt lock, rv=%d lockName=%s lockMode=%s"),
+ status, name, NULLSTR(virLockManagerDlmToModeText(mode)));
+ goto cleanup;
+ }
+
+ if (!virLockManagerDlmRecordLock(name, mode, lksb.sb_lkid, vm_pid)) {
+ virReportSystemError(errno,
+ _("unable to record lock information, "
+ "lockName=%s lockMode=%s lockId=%d vm_pid=%jd"),
+ NULLSTR(name), NULLSTR(virLockManagerDlmToModeText(mode)),
+ lksb.sb_lkid, (intmax_t)vm_pid);
+ }
+
+
+ cleanup:
+ if (name)
+ VIR_FREE(name);
+
+ return;
+}
+
+static int virLockManagerDlmPrepareLockList(const char *lockRecordFilePath)
+{
+ FILE *fp = NULL;
+ int line = 0;
+ size_t n = 0;
+ ssize_t count = 0;
+ char *buffer = NULL;
+
+ fp = fopen(lockRecordFilePath, "r");
+ if (!fp) {
+ virReportSystemError(errno,
+ _("unable to open '%s'"), lockRecordFilePath);
+ return -1;
+ }
+
+ /* lock information is from the second line */
+ for (line = 0; !feof(fp); line++) {
+ count = getline(&buffer, &n, fp);
+ if (count <= 0)
+ break;
+
+ switch (line) {
+ case 0:
+ break;
+ default:
+ virLockManagerDlmAdoptLock(buffer);
+ break;
+ }
+ }
+
+ VIR_FORCE_FCLOSE(fp);
+ VIR_FREE(buffer);
+
+ return 0;
+}
+
+static int virLockManagerDlmGetLocalNodeId(uint32_t *nodeId)
+{
+ cpg_handle_t handle = 0;
+ int rv = -1;
+
+ if (cpg_model_initialize(&handle, CPG_MODEL_V1, NULL, NULL) != CS_OK) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("unable to create a new connection to the CPG service"));
+ return -1;
+ }
+
+ if( cpg_local_get(handle, nodeId) != CS_OK) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("unable to get the local node id by the CPG service"));
+ goto cleanup;
+ }
+
+ VIR_DEBUG("the local nodeid=%u", *nodeId);
+
+ rv = 0;
+
+ cleanup:
+ if (cpg_finalize(handle) != CS_OK)
+ VIR_WARN("unable to finalize the CPG service");
+
+ return rv;
+}
+
+static int virLockManagerDlmDumpLockList(const char *lockRecordFilePath)
+{
+ virLockInformationPtr theLock = NULL;
+ char buffer[BUFFERLEN] = {0};
+ int fd = -1, rv = -1;
+
+ /* not need mutex because of only one instance would be initialized */
+ fd = open(lockRecordFilePath, O_WRONLY|O_CREAT|O_TRUNC, LOCK_RECORD_FILE_MODE);
+ if (fd < 0) {
+ virReportSystemError(errno,
+ _("unable to open '%s'"),
+ lockRecordFilePath);
+ return -1;
+ }
+
+ snprintf(buffer, sizeof(buffer), "%6s %64s %9s %10s\n", \
+ STATUS, RESOURCE_NAME, LOCK_MODE, VM_PID);
+ if (safewrite(fd, buffer, strlen(buffer)) != strlen(buffer)) {
+ virReportSystemError(errno,
+ _("unable to write '%s' to '%s'"),
+ buffer, lockRecordFilePath);
+ goto cleanup;
+ }
+
+ virListForEachEntry(theLock, &(lockListWait.list), entry) {
+ virLockManagerDlmWriteLock(theLock, fd, 1);
+ }
+
+ if (VIR_CLOSE(fd) < 0) {
+ virReportSystemError(errno,
+ _("unable to close file '%s'"),
+ lockRecordFilePath);
+ goto cleanup;
+ }
+
+ rv = 0;
+
+ cleanup:
+ if (rv)
+ VIR_FORCE_CLOSE(fd);
+ return rv;
+}
+
+static int virLockManagerDlmSetupLockRecordFile(const char *lockRecordFilePath,
+ const bool newLockspace,
+ const bool purgeLockspace)
+{
+ uint32_t nodeId = 0;
+
+ /* there maybe some orphan locks recorded in the lock record file which
+ * should be adopted if lockspace is opened instead of created, we adopt
+ * them then add them in the list.
+ */
+ if (!newLockspace &&
+ virLockManagerDlmPrepareLockList(lockRecordFilePath)) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("unable to adopt locks from '%s'"),
+ NULLSTR(lockRecordFilePath));
+ return -1;
+ }
+
+ /* purgeLockspace flag means purging orphan locks belong to any process
+ * in this lockspace.
+ */
+ if (purgeLockspace && !virLockManagerDlmGetLocalNodeId(&nodeId)) {
+ if (dlm_ls_purge(lockspace, nodeId, 0)) {
+ VIR_WARN("node=%u purge DLM locks failed in lockspace=%s",
+ nodeId, NULLSTR(driver->lockspaceName));
+ }
+ else
+ VIR_DEBUG("node=%u purge DLM locks success in lockspace=%s",
+ nodeId, NULLSTR(driver->lockspaceName));
+ }
+
+ /* initialize the lock record file */
+ if (virLockManagerDlmDumpLockList(lockRecordFilePath)) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("unable to initialize the lock record file '%s'"),
+ lockRecordFilePath);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int virLockManagerDlmSetup(void)
+{
+ bool newLockspace = false;
+
+ virListHeadInit(&(lockListWait.list));
+ if ((virMutexInit(&(lockListWait.listMutex)) < 0) ||
+ (virMutexInit(&(lockListWait.fileMutex)) < 0)) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("unable to initialize mutex"));
+ return -1;
+ }
+
+
+ /* check whether dlm is running or not */
+ if (access(DLM_CLUSTER_NAME_PATH, F_OK)) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("check dlm_controld, ensure it has setuped"));
+ return -1;
+ }
+
+ /* open lockspace, create it if it doesn't exist */
+ lockspace = dlm_open_lockspace(driver->lockspaceName);
+ if (!lockspace) {
+ lockspace = dlm_create_lockspace(driver->lockspaceName, DLM_LOCKSPACE_MODE);
+ if (!lockspace) {
+ virReportSystemError(errno, "%s",
+ _("unable to open and create DLM lockspace"));
+ return -1;
+ }
+ newLockspace = true;
+ }
+
+ /* create thread to receive notification from kernel */
+ if (dlm_ls_pthread_init(lockspace)) {
+ if (errno != EEXIST) {
+ virReportSystemError(errno, "%s",
+ _("unable to initialize lockspace"));
+ return -1;
+ }
+ }
+
+ /* we need file to record lock information used by rebooted libvirtd */
+ if (virLockManagerDlmSetupLockRecordFile(driver->lockRecordFilePath,
+ newLockspace,
+ driver->purgeLockspace)) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("unable to initialize DLM lock file"));
+ return -1;
+ }
+
+ return 0;
+}
+
+static int virLockManagerDlmDeinit(void);
+
+static int virLockManagerDlmInit(unsigned int version,
+ const char *configFile,
+ unsigned int flags)
+{
+ VIR_DEBUG("version=%u configFile=%s flags=0x%x", version, NULLSTR(configFile), flags);
+
+ virCheckFlags(0, -1);
+
+ if (driver)
+ return 0;
+
+ if (geteuid() != 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("dlm lock requires root privileges"));
+ return -1;
+ }
+
+ if (VIR_ALLOC(driver) < 0)
+ return -1;
+
+ driver->autoDiskLease = true;
+ driver->requireLeaseForDisks = !driver->autoDiskLease;
+ driver->purgeLockspace = true;
+
+ if (virAsprintf(&driver->lockspaceName,
+ "%s", DLM_LOCKSPACE_NAME) < 0)
+ goto error;
+
+ if (virAsprintf(&driver->lockRecordFilePath,
+ "%s", LOCK_RECORD_FILE_PATH) < 0)
+ goto error;
+
+ if (virLockManagerDlmLoadConfig(configFile) < 0)
+ goto error;
+
+ if (virLockManagerDlmSetup() < 0)
+ goto error;
+
+ return 0;
+
+ error:
+ virLockManagerDlmDeinit();
+ return -1;
+}
+
+static int virLockManagerDlmDeinit(void)
+{
+ virLockInformationPtr theLock = NULL;
+
+ if (!driver)
+ return 0;
+
+ if(lockspace)
+ dlm_close_lockspace(lockspace);
+
+ /* not care about whether adopting lock or not,
+ * just release those to prevent memory leak
+ */
+ virListForEachEntry(theLock, &(lockListWait.list), entry) {
+ virListDelete(&(theLock->entry));
+ VIR_FREE(theLock->name);
+ VIR_FREE(theLock);
+ }
+
+ VIR_FREE(driver->lockspaceName);
+ VIR_FREE(driver->lockRecordFilePath);
+ VIR_FREE(driver);
+
+ return 0;
+}
+
+static int virLockManagerDlmNew(virLockManagerPtr lock,
+ unsigned int type,
+ size_t nparams,
+ virLockManagerParamPtr params,
+ unsigned int flags)
+{
+ virLockManagerDlmPrivatePtr priv = NULL;
+ size_t i;
+
+ virCheckFlags(VIR_LOCK_MANAGER_NEW_STARTED, -1);
+
+ if (!driver) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("dlm plugin is not initialized"));
+ return -1;
+ }
+
+ if (type != VIR_LOCK_MANAGER_OBJECT_TYPE_DOMAIN) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("unsupported object type %d"), type);
+ return -1;
+ }
+
+ if (VIR_ALLOC(priv) < 0)
+ return -1;
+
+ for (i = 0; i< nparams; i++) {
+ if (STREQ(params[i].key, "uuid")) {
+ memcpy(priv->vm_uuid, params[i].value.uuid, VIR_UUID_BUFLEN);
+ } else if (STREQ(params[i].key, "name")) {
+ if (VIR_STRDUP(priv->vm_name, params[i].value.str) < 0)
+ return -1;
+ } else if (STREQ(params[i].key, "id")) {
+ priv->vm_id = params[i].value.ui;
+ } else if (STREQ(params[i].key, "pid")) {
+ priv->vm_pid = params[i].value.iv;
+ } else if (STREQ(params[i].key, "uri")) {
+ /* there would be a warning in some case according to the history patch,
+ * so ignored
+ */
+ } else {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("unexpected parameter %s for object"),
+ params[i].key);
+ }
+ }
+
+ /* check the following to prevent some unexpexted state in some case */
+ if (priv->vm_pid == 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing PID parameter for domain object"));
+ return -1;
+ }
+ if (!priv->vm_name) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing name parameter for domain object"));
+ return -1;
+ }
+
+ if (priv->vm_id == 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing ID parameter for domain object"));
+ return -1;
+ }
+ if (!virUUIDIsValid(priv->vm_uuid)) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing UUID parameter for domain object"));
+ return -1;
+ }
+
+ lock->privateData = priv;
+
+ return 0;
+}
+
+static void virLockManagerDlmFree(virLockManagerPtr lock)
+{
+ virLockManagerDlmPrivatePtr priv = lock->privateData;
+ size_t i;
+
+ if (!priv)
+ return;
+
+ for (i = 0; i < priv->nresources; i++)
+ VIR_FREE(priv->resources[i].name);
+
+ VIR_FREE(priv->resources);
+ VIR_FREE(priv->vm_name);
+ VIR_FREE(priv);
+ lock->privateData = NULL;
+
+ return;
+}
+
+static int virLockManagerDlmAddResource(virLockManagerPtr lock,
+ unsigned int type, const char *name,
+ size_t nparams,
+ virLockManagerParamPtr params,
+ unsigned int flags)
+{
+ virLockManagerDlmPrivatePtr priv = lock->privateData;
+ char *newName = NULL;
+
+ virCheckFlags(VIR_LOCK_MANAGER_RESOURCE_READONLY |
+ VIR_LOCK_MANAGER_RESOURCE_SHARED, -1);
+
+ /* Treat read only resources as a no-op lock request */
+ if (flags & VIR_LOCK_MANAGER_RESOURCE_READONLY)
+ return 0;
+
+ switch (type) {
+ case VIR_LOCK_MANAGER_RESOURCE_TYPE_DISK:
+ if (params || nparams) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("unexpected parameters for disk resource"));
+ return -1;
+ }
+
+ if (!driver->autoDiskLease) {
+ if (!(flags & (VIR_LOCK_MANAGER_RESOURCE_SHARED |
+ VIR_LOCK_MANAGER_RESOURCE_READONLY))) {
+ priv->hasRWDisks = true;
+ /* ignore disk resource without error */
+ return 0;
+ }
+ }
+
+ if (virCryptoHashString(VIR_CRYPTO_HASH_SHA256, name, &newName) < 0)
+ goto cleanup;
+
+ break;
+
+ case VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE:
+ /* we need format the lock information, so the lock name must be the constant length */
+ if (virCryptoHashString(VIR_CRYPTO_HASH_SHA256, name, &newName) < 0)
+ goto cleanup;
+
+ break;
+
+ default:
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("unknown lock manager object type %d"),
+ type);
+ return -1;
+ }
+
+ if (VIR_EXPAND_N(priv->resources, priv->nresources, 1) < 0)
+ goto cleanup;
+
+ priv->resources[priv->nresources-1].name = newName;
+
+ if (!!(flags & VIR_LOCK_MANAGER_RESOURCE_SHARED))
+ priv->resources[priv->nresources-1].mode = LKM_PRMODE;
+ else
+ priv->resources[priv->nresources-1].mode = LKM_EXMODE;
+
+ return 0;
+
+ cleanup:
+ VIR_FREE(newName);
+
+ return -1;
+}
+
+static int virLockManagerDlmAcquire(virLockManagerPtr lock,
+ const char *state ATTRIBUTE_UNUSED,
+ unsigned int flags,
+ virDomainLockFailureAction action ATTRIBUTE_UNUSED,
+ int *fd)
+{
+ virLockManagerDlmPrivatePtr priv = lock->privateData;
+ virLockInformationPtr theLock = NULL;
+ struct dlm_lksb lksb = {0};
+ int rv = -1, theFd = -1;
+ size_t i;
+
+ virCheckFlags(VIR_LOCK_MANAGER_ACQUIRE_REGISTER_ONLY |
+ VIR_LOCK_MANAGER_ACQUIRE_RESTRICT, -1);
+
+ /* allowed to start a guest which has read/write disks, but without any leases */
+ if (priv->nresources == 0 &&
+ priv->hasRWDisks &&
+ driver->requireLeaseForDisks) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("read/write, exclusive access, disks were present, but no leases specified"));
+ return -1;
+ }
+
+ /* accorting to git patch history, add `fd` parameter in order to
+ * 'ensure sanlock socket is labelled with the VM process label',
+ * however, fixing sanlock socket security labelling remove related
+ * code. Now, `fd` parameter is useless.
+ */
+ if (fd)
+ *fd = -1;
+
+ if(!lockspace) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("lockspace is not opened"));
+ return -1;
+ }
+
+ if (!(flags & VIR_LOCK_MANAGER_ACQUIRE_REGISTER_ONLY)) {
+ VIR_DEBUG("Acquiring object %zu", priv->nresources);
+
+ theFd = open(driver->lockRecordFilePath, O_RDWR);
+ if (theFd < 0) {
+ virReportSystemError(errno,
+ _("unable to open '%s'"), driver->lockRecordFilePath);
+ return -1;
+ }
+
+ for (i = 0; i < priv->nresources; i++) {
+ VIR_DEBUG("Acquiring object %zu", priv->nresources);
+
+ memset(&lksb, 0, sizeof(lksb));
+ rv = dlm_ls_lock_wait(lockspace, priv->resources[i].mode,
+ &lksb, LKF_NOQUEUE|LKF_PERSISTENT,
+ priv->resources[i].name, strlen(priv->resources[i].name),
+ 0, NULL, NULL, NULL);
+ /* both `rv` and `lksb.sb_status` equal 0 means lock sucessfully */
+ if (rv || lksb.sb_status) {
+ if (lksb.sb_status == EAGAIN)
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("failed to acquire lock: the lock could not be granted"));
+ else {
+ virReportSystemError(errno,
+ _("failed to acquire lock: rv=%d lockStatus=%d"),
+ rv, lksb.sb_status);
+ }
+ /* rv would be 0 although acquiring lock failed */
+ rv = -1;
+ goto cleanup;
+ }
+
+ theLock = virLockManagerDlmRecordLock(priv->resources[i].name,
+ priv->resources[i].mode,
+ lksb.sb_lkid,
+ priv->vm_pid);
+ if (!theLock) {
+ virReportSystemError(errno,
+ _("unable to record lock information, "
+ "lockName=%s lockMode=%s lockId=%d vm_pid=%jd"),
+ NULLSTR(priv->resources[i].name),
+ NULLSTR(virLockManagerDlmToModeText(priv->resources[i].mode)),
+ lksb.sb_lkid, (intmax_t)priv->vm_pid);
+ /* record lock failed, we can't save lock information in memory, so release it */
+ rv = dlm_ls_unlock_wait(lockspace, lksb.sb_lkid, 0, &lksb);
+ if (!rv)
+ virReportSystemError(errno,
+ _("failed to release lock: rv=%d lockStatue=%d"),
+ rv, lksb.sb_status);
+ rv = -1;
+ goto cleanup;
+ }
+
+ virMutexLock(&(lockListWait.fileMutex));
+ virLockManagerDlmWriteLock(theLock, theFd, 1);
+ virMutexUnlock(&(lockListWait.fileMutex));
+ }
+
+ if(VIR_CLOSE(theFd) < 0) {
+ virReportSystemError(errno,
+ _("unable to save file '%s'"),
+ driver->lockRecordFilePath);
+ goto cleanup;
+ }
+ }
+
+ if (flags & VIR_LOCK_MANAGER_ACQUIRE_RESTRICT) {
+ /* no daemon watches this fd, do nothing here, just close lockspace before `execv`
+ * `dlm_close_lockspace` always return 0, so ignore return value
+ */
+ ignore_value(dlm_close_lockspace(lockspace));
+ lockspace = NULL;
+ }
+
+ rv = 0;
+
+ cleanup:
+ if (rv)
+ VIR_FORCE_CLOSE(theFd);
+ return rv;
+}
+
+static void virLockManagerDlmDeleteLock(const virLockInformationPtr lock,
+ const char *lockRecordFilePath)
+{
+ int fd = -1;
+
+ if (!lock)
+ return;
+
+ virMutexLock(&(lockListWait.listMutex));
+ virListDelete(&(lock->entry));
+ virMutexUnlock(&(lockListWait.listMutex));
+
+ fd = open(lockRecordFilePath, O_RDWR);
+ if (fd < 0) {
+ virReportSystemError(errno,
+ _("unable to open '%s'"), lockRecordFilePath);
+ goto cleanup;
+ }
+
+ virMutexLock(&(lockListWait.fileMutex));
+ virLockManagerDlmWriteLock(lock, fd, 0);
+ virMutexUnlock(&(lockListWait.fileMutex));
+
+ if (VIR_CLOSE(fd) < 0) {
+ virReportSystemError(errno,
+ _("unable to save file '%s'"),
+ lockRecordFilePath);
+ VIR_FORCE_CLOSE(fd);
+ }
+
+ cleanup:
+ VIR_FREE(lock->name);
+ VIR_FREE(lock);
+}
+
+static int virLockManagerDlmRelease(virLockManagerPtr lock,
+ char **state,
+ unsigned int flags)
+{
+ virLockManagerDlmPrivatePtr priv = lock->privateData;
+ virLockManagerDlmResourcePtr resource = NULL;
+ virLockInformationPtr theLock = NULL;
+ struct dlm_lksb lksb = {0};
+ int rv = -1;
+ size_t i;
+
+ virCheckFlags(0, -1);
+
+ if(state)
+ *state = NULL;
+
+ if(!lockspace) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("lockspace is not opened"));
+ return -1;
+ }
+
+ for (i = 0; i < priv->nresources; i++) {
+ resource = priv->resources + i;
+
+ virListForEachEntry (theLock, &(lockListWait.list), entry) {
+ if((theLock->vm_pid == priv->vm_pid) &&
+ STREQ(theLock->name, resource->name) &&
+ (theLock->mode == resource->mode)) {
+
+ /*
+ * there are some locks from adopting, the existence of `(void *)1`
+ * when adopting makes 'terminated by signal SIGSEGV (Address
+ * boundary error)' error appear.
+ *
+ * The following code reference to lvm2 project's implement.
+ */
+ lksb.sb_lkid = theLock->lkid;
+ rv = dlm_ls_lock_wait(lockspace, LKM_NLMODE,
+ &lksb, LKF_CONVERT,
+ resource->name,
+ strlen(resource->name),
+ 0, NULL, NULL, NULL);
+
+ if (rv < 0) {
+ virReportSystemError(errno,
+ _("failed to convert lock: rv=%d lockStatus=%d"),
+ rv, lksb.sb_status);
+ goto cleanup;
+ }
+
+ /* don't care whether the lock is released or not,
+ * it will be automatically released after the libvirtd dead
+ */
+ virLockManagerDlmDeleteLock(theLock, driver->lockRecordFilePath);
+
+ rv = dlm_ls_unlock_wait(lockspace, lksb.sb_lkid, 0, &lksb);
+ if (rv < 0) {
+ virReportSystemError(errno,
+ _("failed to release lock: rv=%d lockStatus=%d"),
+ rv, lksb.sb_status);
+ goto cleanup;
+ }
+
+ break;
+ }
+ }
+ }
+
+ rv = 0;
+
+ cleanup:
+ return rv;
+}
+
+static int virLockManagerDlmInquire(virLockManagerPtr lock ATTRIBUTE_UNUSED,
+ char **state,
+ unsigned int flags)
+{
+ /* not support mannual lock, so this function almost does nothing */
+ virCheckFlags(0, -1);
+
+ if (state)
+ *state = NULL;
+
+ return 0;
+}
+
+virLockDriver virLockDriverImpl =
+{
+ .version = VIR_LOCK_MANAGER_VERSION,
+
+ .flags = VIR_LOCK_MANAGER_USES_STATE, // currently not used
+
+ .drvInit = virLockManagerDlmInit,
+ .drvDeinit = virLockManagerDlmDeinit,
+
+ .drvNew = virLockManagerDlmNew,
+ .drvFree = virLockManagerDlmFree,
+
+ .drvAddResource = virLockManagerDlmAddResource,
+
+ .drvAcquire = virLockManagerDlmAcquire,
+ .drvRelease = virLockManagerDlmRelease,
+ .drvInquire = virLockManagerDlmInquire,
+};
diff --git a/src/util/virlist.h b/src/util/virlist.h
new file mode 100644
index 000000000..4ac3626c7
--- /dev/null
+++ b/src/util/virlist.h
@@ -0,0 +1,110 @@
+/*
+ * virlist.h: methods for managing list, logic is copied from Linux Kernel
+ *
+ * Copyright (C) 2018 SUSE LINUX Products, Beijing, China.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef __VIR_LIST_H
+#define __VIR_LIST_H
+
+#include <stdbool.h>
+
+typedef struct _virListHead virListHead;
+typedef virListHead *virListHeadPtr;
+
+struct _virListHead {
+ struct _virListHead *next, *prev;
+};
+
+static inline void
+virListHeadInit(virListHeadPtr name)
+{
+ name->next = name;
+ name->prev = name;
+}
+
+static inline void
+__virListAdd(virListHeadPtr entry,
+ virListHeadPtr prev, virListHeadPtr next)
+{
+ next->prev = entry;
+ entry->next = next;
+ entry->prev = prev;
+ prev->next = entry;
+}
+
+static inline void
+virListAdd(virListHeadPtr entry, virListHeadPtr head)
+{
+ __virListAdd(entry, head, head->next);
+}
+
+static inline void
+virListAddTail(virListHeadPtr entry, virListHeadPtr head)
+{
+ __virListAdd(entry, head->prev, head);
+}
+
+static inline void
+__virListDelete(virListHeadPtr prev, virListHeadPtr next)
+{
+ next->prev = prev;
+ prev->next = next;
+}
+
+static inline void
+virListDelete(virListHeadPtr entry)
+{
+ __virListDelete(entry->prev, entry->next);
+}
+
+static inline bool
+virListEmpty(virListHeadPtr head)
+{
+ return head->next == head;
+}
+
+#ifndef virContainerOf
+#define virContainerOf(ptr, type, member) \
+ (type *)((char *)(ptr) - (char *) &((type *)0)->member)
+#endif
+
+#define virListEntry(ptr, type, member) \
+ virContainerOf(ptr, type, member)
+
+#define virListFirstEntry(ptr, type, member) \
+ virListEntry((ptr)->next, type, member)
+
+#define virListLastEntry(ptr, type, member) \
+ virListEntry((ptr)->prev, type, member)
+
+#define __virContainerOf(ptr, sample, member) \
+ (void *)virContainerOf((ptr), typeof(*(sample)), member)
+
+#define virListForEachEntry(pos, head, member) \
+ for (pos = __virContainerOf((head)->next, pos, member); \
+ &pos->member != (head); \
+ pos = __virContainerOf(pos->member.next, pos, member))
+
+#define virListForEachEntrySafe(pos, tmp, head, member) \
+ for (pos = __virContainerOf((head)->next, pos, member), \
+ tmp = __virContainerOf(pos->member.next, pos, member); \
+ &pos->member != (head); \
+ pos = tmp, tmp = __virContainerOf(pos->member.next, tmp, member))
+
+#endif
--
2.15.1
More information about the libvir-list
mailing list