[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [libvirt] [PATCH 3/3] Support automatic creation of leases for disks in sanlock



On Fri, Jun 17, 2011 at 01:38:21PM +0100, Daniel P. Berrange wrote:
> The current sanlock plugin requires a central management
> application to manually add <lease> elements to each guest,
> to protect resources that are assigned to it (eg writable
> disks). This makes the sanlock plugin useless for usage
> in more adhoc deployment environments where there is no
> central authority to associate disks with leases.
> 
> This patch adds a mode where the sanlock plugin will
> automatically create leases for each assigned read-write
> disk, using a md5 checksum of the fully qualified disk
> path. This can work pretty well if guests are using
> stable disk paths for block devices eg /dev/disk/by-path/XXXX
> symlinks, or if all hosts have NFS volumes mounted in
> a consistent pattern.
> 
> The plugin will create one lockspace for managing disks
> with filename /var/lib/libvirt/sanlock/__LIBVIRT__DISKS__.
> For each VM disks, there will be another file to hold
> a lease /var/lib/libvirt/sanlock/5903e5d25e087e60a20fe4566fab41fd
> Each VM disk lease is usually 1 MB in size. A nightly
> cron job will delete any unused lease files from the
> lockspace directory.
> 
> To make use of this capability the admin will need todo
> several tasks:
> 
>  - Mount an NFS volume (or other shared filesystem)
>    on /var/lib/libvirt/sanlock
>  - Configure 'host_id' in /etc/libvirt/qemu-sanlock.conf
>    with a unique value for each host with the same NFS
>    mount
>  - Toggle the 'auto_disk_leases' parameter in qemu-sanlock.conf
> 
> Technically the first step can be skipped, in which case
> sanlock will only protect against 2 vms on the same host
> using the same disk (or the same VM being started twice
> due to error by libvirt).
> 
> * src/locking/libvirt_sanlock.aug,
>   src/locking/sanlock.conf,
>   src/locking/test_libvirt_sanlock.aug: Add config params
>   for configuring auto lease setup
> * libvirt.spec.in: Add virt-sanlock-cleanup program, man
>   page and cronjob
> * tools/virt-sanlock-cleanup.cron.in: Nightly cron job
> * tools/virt-sanlock-cleanup.in: Script to purge unused
>   disk resource lease files
> ---
>  libvirt.spec.in                      |    3 +
>  src/locking/libvirt_sanlock.aug      |    6 +-
>  src/locking/lock_driver_sanlock.c    |  449 ++++++++++++++++++++++++++++++---
>  src/locking/sanlock.conf             |   60 +++++
>  src/locking/test_libvirt_sanlock.aug |   10 +-
>  tools/.gitignore                     |    3 +
>  tools/Makefile.am                    |   25 ++-
>  tools/virt-sanlock-cleanup.cron.in   |    4 +
>  tools/virt-sanlock-cleanup.in        |   91 +++++++
>  9 files changed, 607 insertions(+), 44 deletions(-)
>  create mode 100644 tools/virt-sanlock-cleanup.cron.in
>  create mode 100644 tools/virt-sanlock-cleanup.in
> 
> diff --git a/libvirt.spec.in b/libvirt.spec.in
> index a8d7026..8f26876 100644
> --- a/libvirt.spec.in
> +++ b/libvirt.spec.in
> @@ -1025,9 +1025,12 @@ fi
>  %files lock-sanlock
>  %defattr(-, root, root)
>  %config(noreplace) %{_sysconfdir}/libvirt/qemu-sanlock.conf
> +%{_sysconfdir}/cron.daily/virt-sanlock-cleanup
>  %attr(0755, root, root) %{_libdir}/libvirt/lock-driver/sanlock.so
>  %{_datadir}/augeas/lenses/libvirt_sanlock.aug
>  %{_datadir}/augeas/lenses/tests/test_libvirt_sanlock.aug
> +%{_sbindir}/virt-sanlock-cleanup
> +%{_mandir}/man8/virt-sanlock-cleanup.8*
>  %endif
>  
>  %files client -f %{name}.lang
> diff --git a/src/locking/libvirt_sanlock.aug b/src/locking/libvirt_sanlock.aug
> index baa639a..548181e 100644
> --- a/src/locking/libvirt_sanlock.aug
> +++ b/src/locking/libvirt_sanlock.aug
> @@ -17,7 +17,11 @@ module Libvirt_sanlock =
>  
>  
>     (* Each enty in the config is one of the following three ... *)
> -   let entry = bool_entry "require_lease_for_disks"
> +   let entry = str_entry "disk_lease_dir"
> +             | bool_entry "auto_disk_leases"
> +             | int_entry "max_hosts"
> +             | int_entry "host_id"
> +             | bool_entry "require_lease_for_disks"
>     let comment = [ label "#comment" . del /#[ \t]*/ "# " .  store /([^ \t\n][^\n]*)?/ . del /\n/ "\n" ]
>     let empty = [ label "#empty" . eol ]
>  
> diff --git a/src/locking/lock_driver_sanlock.c b/src/locking/lock_driver_sanlock.c
> index 7ad54e8..50fa9cc 100644
> --- a/src/locking/lock_driver_sanlock.c
> +++ b/src/locking/lock_driver_sanlock.c
> @@ -28,9 +28,13 @@
>  #include <stdio.h>
>  #include <errno.h>
>  #include <sys/types.h>
> +#include <sys/stat.h>
> +#include <fcntl.h>
>  
>  #include <sanlock.h>
>  #include <sanlock_resource.h>
> +#include <sanlock_direct.h>
> +#include <sanlock_admin.h>
>  
>  #include "lock_driver.h"
>  #include "logging.h"
> @@ -38,6 +42,10 @@
>  #include "memory.h"
>  #include "util.h"
>  #include "files.h"
> +#include "md5.h"
> +#include "conf.h"
> +
> +#include "configmake.h"
>  
>  #define VIR_FROM_THIS VIR_FROM_LOCKING
>  
> @@ -46,6 +54,8 @@
>                           __FUNCTION__, __LINE__, __VA_ARGS__)
>  
>  
> +#define VIR_LOCK_MANAGER_SANLOCK_AUTO_DISK_LOCKSPACE "__LIBVIRT__DISKS__"
> +
>  typedef struct _virLockManagerSanlockDriver virLockManagerSanlockDriver;
>  typedef virLockManagerSanlockDriver *virLockManagerSanlockDriverPtr;
>  
> @@ -54,6 +64,10 @@ typedef virLockManagerSanlockPrivate *virLockManagerSanlockPrivatePtr;
>  
>  struct _virLockManagerSanlockDriver {
>      bool requireLeaseForDisks;
> +    int hostID;
> +    int maxHosts;
> +    bool autoDiskLease;
> +    char *autoDiskLeasePath;
>  };
>  
>  static virLockManagerSanlockDriver *driver = NULL;
> @@ -98,16 +112,155 @@ static int virLockManagerSanlockLoadConfig(const char *configFile)
>          return -1;                                                      \
>      }
>  
> +    p = virConfGetValue(conf, "auto_disk_leases");
> +    CHECK_TYPE("auto_disk_leases", VIR_CONF_LONG);
> +    if (p) driver->autoDiskLease = p->l;
> +
> +    p = virConfGetValue(conf, "disk_lease_dir");
> +    CHECK_TYPE("disk_lease_dir", VIR_CONF_STRING);
> +    if (p && p->str) {
> +        VIR_FREE(driver->autoDiskLeasePath);
> +        if (!(driver->autoDiskLeasePath = strdup(p->str))) {
> +            virReportOOMError();
> +            virConfFree(conf);
> +            return -1;
> +        }
> +    }
> +
> +    p = virConfGetValue(conf, "max_hosts");
> +    CHECK_TYPE("max_hosts", VIR_CONF_LONG);
> +    if (p) driver->maxHosts = p->l;
> +
> +    p = virConfGetValue(conf, "host_id");
> +    CHECK_TYPE("host_id", VIR_CONF_LONG);
> +    if (p) driver->hostID = p->l;
> +
>      p = virConfGetValue(conf, "require_lease_for_disks");
>      CHECK_TYPE("require_lease_for_disks", VIR_CONF_LONG);
>      if (p)
>          driver->requireLeaseForDisks = p->l;
> +    else
> +        driver->requireLeaseForDisks = driver->autoDiskLease ? false : true;
>  
>      virConfFree(conf);
>      return 0;
>  }

  Okay I got the answer w.r.t. the macro use in previous patch

> +static int virLockManagerSanlockSetupLockspace(void)
> +{
> +    int fd = -1;
> +    struct stat st;
> +    int rv;
> +    struct sanlk_lockspace ls;
> +    char *path = NULL;
> +
> +    if (virAsprintf(&path, "%s/%s",
> +                    driver->autoDiskLeasePath,
> +                    VIR_LOCK_MANAGER_SANLOCK_AUTO_DISK_LOCKSPACE) < 0) {
> +        virReportOOMError();
> +        goto error;
> +    }
> +
> +    memcpy(ls.name, VIR_LOCK_MANAGER_SANLOCK_AUTO_DISK_LOCKSPACE, SANLK_NAME_LEN);
> +    ls.host_id = 0; /* Doesn't matter for initialization */
> +    ls.flags = 0;
> +    memcpy(&ls.host_id_disk, path, sizeof(struct sanlk_disk));
> +    ls.host_id_disk.offset = 0;
> +
> +    /* Stage 1: Ensure the lockspace file exists on disk, has
> +     * space allocated for it and is initialized with lease
> +     */
> +    if (stat(path, &st) < 0) {
> +        VIR_DEBUG("Lockspace %s does not yet exist", path);
> +        if ((fd = open(path, O_WRONLY|O_CREAT|O_EXCL, 0600)) < 0) {
> +            if (errno != EEXIST) {
> +                virReportSystemError(errno,
> +                                     _("Unable to create lockspace %s"),
> +                                     path);
> +                goto error;
> +            }
> +            VIR_DEBUG("Someone else just created lockspace %s", path);
> +        } else {
> +            if ((rv = sanlock_direct_align(&ls.host_id_disk)) < 0) {
> +                if (rv <= -200)
> +                    virLockError(VIR_ERR_INTERNAL_ERROR,
> +                                 _("Unable to query sector size %s: error %d"),
> +                                 path, rv);
> +                else
> +                    virReportSystemError(-rv,
> +                                         _("Unable to query sector size %s"),
> +                                         path);
> +                goto error_unlink;
> +            }
> +
> +            /*
> +             * Pre allocate enough data for 1 block of leases at preferred alignment
> +             */
> +            if (safezero(fd, 0, 0, rv) < 0) {
> +                virReportSystemError(errno,
> +                                     _("Unable to allocate lockspace %s"),
> +                                     path);
> +                goto error_unlink;
> +            }
> +
> +            if (VIR_CLOSE(fd) < 0) {
> +                virReportSystemError(errno,
> +                                     _("Unable to save lockspace %s"),
> +                                     path);
> +                goto error_unlink;
> +            }
> +
> +            if ((rv = sanlock_direct_init(&ls, NULL, 0, driver->maxHosts, 0)) < 0) {
> +                if (rv <= -200)
> +                    virLockError(VIR_ERR_INTERNAL_ERROR,
> +                                 _("Unable to initialize lockspace %s: error %d"),
> +                                 path, rv);
> +                else
> +                    virReportSystemError(-rv,
> +                                         _("Unable to initialize lockspace %s"),
> +                                         path);
> +                goto error_unlink;
> +            }
> +            VIR_DEBUG("Lockspace %s has been initialized", path);
> +        }
> +    }
> +
> +    ls.host_id = driver->hostID;
> +    /* Stage 2: Try to register the lockspace with the daemon.
> +     * If the lockspace is already registered, we should get EEXIST back
> +     * in which case we can just carry on with life
> +     */
> +    if ((rv = sanlock_add_lockspace(&ls, 0)) < 0) {
> +        if (-rv != EEXIST) {
> +            if (rv <= -200)
> +                virLockError(VIR_ERR_INTERNAL_ERROR,
> +                             _("Unable to add lockspace %s: error %d"),
> +                             path, rv);
> +            else
> +                virReportSystemError(-rv,
> +                                     _("Unable to add lockspace %s"),
> +                                     path);
> +            return -1;
> +        } else {
> +            VIR_DEBUG("Lockspace %s is already registered", path);
> +        }
> +    } else {
> +        VIR_DEBUG("Lockspace %s has been registered", path);
> +    }
> +
> +    return 0;
> +
> +error_unlink:
> +    if (path)
> +        unlink(path);
> +error:
> +    VIR_FORCE_CLOSE(fd);
> +    VIR_FREE(path);
> +    return -1;
> +}
> +
>  
> +static int virLockManagerSanlockDeinit(void);
>  static int virLockManagerSanlockInit(unsigned int version,
>                                       const char *configFile,
>                                       unsigned int flags)
> @@ -124,11 +277,32 @@ static int virLockManagerSanlockInit(unsigned int version,
>      }
>  
>      driver->requireLeaseForDisks = true;
> +    driver->hostID = 0;
> +    driver->maxHosts = 64;
> +    driver->autoDiskLease = false;
> +    if (!(driver->autoDiskLeasePath = strdup(LOCALSTATEDIR "/lib/libvirt/sanlock"))) {
> +        VIR_FREE(driver);
> +        virReportOOMError();
> +        goto error;
> +    }
>  
>      if (virLockManagerSanlockLoadConfig(configFile) < 0)
> -        return -1;
> +        goto error;
> +
> +    if (driver->autoDiskLease && !driver->hostID) {
> +        virLockError(VIR_ERR_INTERNAL_ERROR,
> +                     _("Automatic disk lease mode enabled, but no host ID is set"));
> +        goto error;
> +    }
> +
> +    if (virLockManagerSanlockSetupLockspace() < 0)
> +        goto error;
>  
>      return 0;
> +
> +error:
> +    virLockManagerSanlockDeinit();
> +    return -1;
>  }
>  
>  static int virLockManagerSanlockDeinit(void)
> @@ -136,9 +310,10 @@ static int virLockManagerSanlockDeinit(void)
>      if (!driver)
>          return 0;
>  
> -    virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
> -                 _("Unloading sanlock plugin is forbidden"));
> -    return -1;
> +    VIR_FREE(driver->autoDiskLeasePath);
> +    VIR_FREE(driver);
> +
> +    return 0;
>  }
>  
>  
> @@ -214,51 +389,50 @@ static void virLockManagerSanlockFree(virLockManagerPtr lock)
>      lock->privateData = NULL;
>  }
>  
> -static int virLockManagerSanlockAddResource(virLockManagerPtr lock,
> -                                            unsigned int type,
> -                                            const char *name,
> -                                            size_t nparams,
> -                                            virLockManagerParamPtr params,
> -                                            unsigned int flags)
> +
> +static const char hex[] = { '0', '1', '2', '3', '4', '5', '6', '7',
> +                            '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
> +
> +static int virLockManagerSanlockDiskLeaseName(const char *path,
> +                                              char *str,
> +                                              size_t strbuflen)
>  {
> -    virLockManagerSanlockPrivatePtr priv = lock->privateData;
> -    struct sanlk_resource *res;
> +    unsigned char buf[MD5_DIGEST_SIZE];
>      int i;
>  
> -    virCheckFlags(VIR_LOCK_MANAGER_RESOURCE_READONLY |
> -                  VIR_LOCK_MANAGER_RESOURCE_SHARED, -1);
> -
> -    if (priv->res_count == SANLK_MAX_RESOURCES) {
> +    if (strbuflen < ((MD5_DIGEST_SIZE * 2) + 1)) {
>          virLockError(VIR_ERR_INTERNAL_ERROR,
> -                     _("Too many resources %d for object"),
> -                     SANLK_MAX_RESOURCES);
> +                     _("String length too small to store md5 checksum"));
>          return -1;
>      }
>  
> -    if (type == VIR_LOCK_MANAGER_RESOURCE_TYPE_DISK) {
> -        if (!(flags & (VIR_LOCK_MANAGER_RESOURCE_SHARED |
> -                       VIR_LOCK_MANAGER_RESOURCE_READONLY)))
> -            priv->hasRWDisks = true;
> -        return 0;
> -    }
> -
> -    if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE)
> -        return 0;
> -
> -    if (flags & VIR_LOCK_MANAGER_RESOURCE_READONLY) {
> -        virLockError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> -                     _("Readonly leases are not supported"));
> +    if (!(md5_buffer(path, strlen(path), buf))) {
> +        virLockError(VIR_ERR_INTERNAL_ERROR,
> +                     _("Unable to compute md5 checksum"));
>          return -1;
>      }
> -    if (flags & VIR_LOCK_MANAGER_RESOURCE_SHARED) {
> -        virLockError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> -                     _("Sharable leases are not supported"));
> -        return -1;
> +
> +    for (i = 0 ; i < MD5_DIGEST_SIZE ; i++) {
> +        str[i*2] = hex[(buf[i] >> 4) & 0xf];
> +        str[(i*2)+1] = hex[buf[i] & 0xf];
>      }
> +    str[(MD5_DIGEST_SIZE*2)+1] = '\0';
> +    return 0;
> +}
> +
> +static int virLockManagerSanlockAddLease(virLockManagerPtr lock,
> +                                         const char *name,
> +                                         size_t nparams,
> +                                         virLockManagerParamPtr params)
> +{
> +    virLockManagerSanlockPrivatePtr priv = lock->privateData;
> +    int ret = -1;
> +    struct sanlk_resource *res = NULL;
> +    int i;
>  
>      if (VIR_ALLOC_VAR(res, struct sanlk_disk, 1) < 0) {
>          virReportOOMError();
> -        return -1;
> +        goto cleanup;
>      }
>  
>      res->num_disks = 1;
> @@ -266,7 +440,7 @@ static int virLockManagerSanlockAddResource(virLockManagerPtr lock,
>          virLockError(VIR_ERR_INTERNAL_ERROR,
>                       _("Resource name '%s' exceeds %d characters"),
>                       name, SANLK_NAME_LEN);
> -        goto error;
> +        goto cleanup;
>      }
>  
>      for (i = 0; i < nparams; i++) {
> @@ -275,7 +449,7 @@ static int virLockManagerSanlockAddResource(virLockManagerPtr lock,
>                  virLockError(VIR_ERR_INTERNAL_ERROR,
>                               _("Lease path '%s' exceeds %d characters"),
>                               params[i].value.str, SANLK_PATH_LEN);
> -                goto error;
> +                goto cleanup;
>              }
>          } else if (STREQ(params[i].key, "offset")) {
>              res->disks[0].offset = params[i].value.ul;
> @@ -284,20 +458,213 @@ static int virLockManagerSanlockAddResource(virLockManagerPtr lock,
>                  virLockError(VIR_ERR_INTERNAL_ERROR,
>                               _("Resource lockspace '%s' exceeds %d characters"),
>                               params[i].value.str, SANLK_NAME_LEN);
> -                goto error;
> +                goto cleanup;
>              }
>          }
>      }
>  
>      priv->res_args[priv->res_count] = res;
>      priv->res_count++;
> +
> +    ret = 0;
> +
> +cleanup:
> +    if (ret == -1)
> +        VIR_FREE(res);
> +    return ret;
> +}
> +
> +
> +
> +
> +static int virLockManagerSanlockAddDisk(virLockManagerPtr lock,
> +                                        const char *name,
> +                                        size_t nparams,
> +                                        virLockManagerParamPtr params ATTRIBUTE_UNUSED)
> +{
> +    virLockManagerSanlockPrivatePtr priv = lock->privateData;
> +    int ret = -1;
> +    struct sanlk_resource *res = NULL;
> +    char *path = NULL;
> +
> +    if (nparams) {
> +        virLockError(VIR_ERR_INTERNAL_ERROR,
> +                     _("Unexpected lock parameters for disk resource"));
> +        return -1;
> +    }
> +
> +    if (VIR_ALLOC_VAR(res, struct sanlk_disk, 1) < 0) {
> +        virReportOOMError();
> +        goto cleanup;
> +    }
> +
> +    res->num_disks = 1;
> +    if (virLockManagerSanlockDiskLeaseName(name, res->name, SANLK_NAME_LEN) < 0)
> +        goto cleanup;
> +
> +    if (virAsprintf(&path, "%s/%s",
> +                    driver->autoDiskLeasePath, res->name) < 0) {
> +        virReportOOMError();
> +        goto cleanup;
> +    }
> +    if (!virStrcpy(res->disks[0].path, path, SANLK_PATH_LEN)) {
> +        virLockError(VIR_ERR_INTERNAL_ERROR,
> +                     _("Lease path '%s' exceeds %d characters"),
> +                     path, SANLK_PATH_LEN);
> +        goto cleanup;
> +    }
> +
> +    if (!virStrcpy(res->lockspace_name,
> +                   VIR_LOCK_MANAGER_SANLOCK_AUTO_DISK_LOCKSPACE,
> +                   SANLK_NAME_LEN)) {
> +        virLockError(VIR_ERR_INTERNAL_ERROR,
> +                     _("Resource lockspace '%s' exceeds %d characters"),
> +                     VIR_LOCK_MANAGER_SANLOCK_AUTO_DISK_LOCKSPACE, SANLK_NAME_LEN);
> +        goto cleanup;
> +    }
> +
> +    priv->res_args[priv->res_count] = res;
> +    priv->res_count++;
> +
> +    ret = 0;
> +
> +cleanup:
> +    if (ret == -1)
> +        VIR_FREE(res);
> +    VIR_FREE(path);
> +    return ret;
> +}
> +
> +
> +static int virLockManagerSanlockCreateLease(struct sanlk_resource *res)
> +{
> +    int fd = -1;
> +    struct stat st;
> +    int rv;
> +
> +    if (stat(res->disks[0].path, &st) < 0) {
> +        VIR_DEBUG("Lockspace %s does not yet exist", res->disks[0].path);
> +        if ((fd = open(res->disks[0].path, O_WRONLY|O_CREAT|O_EXCL, 0600)) < 0) {
> +            if (errno != EEXIST) {
> +                virReportSystemError(errno,
> +                                     _("Unable to create lockspace %s"),
> +                                     res->disks[0].path);
> +                return -1;
> +            }
> +            VIR_DEBUG("Someone else just created lockspace %s", res->disks[0].path);
> +        } else {
> +            if ((rv = sanlock_direct_align(&res->disks[0])) < 0) {
> +                if (rv <= -200)
> +                    virLockError(VIR_ERR_INTERNAL_ERROR,
> +                                 _("Unable to query sector size %s: error %d"),
> +                                 res->disks[0].path, rv);
> +                else
> +                    virReportSystemError(-rv,
> +                                         _("Unable to query sector size %s"),
> +                                         res->disks[0].path);
> +                goto error_unlink;
> +            }
> +
> +            /*
> +             * Pre allocate enough data for 1 block of leases at preferred alignment
> +             */
> +            if (safezero(fd, 0, 0, rv) < 0) {
> +                virReportSystemError(errno,
> +                                     _("Unable to allocate lease %s"),
> +                                     res->disks[0].path);
> +                goto error_unlink;
> +            }
> +
> +            if (VIR_CLOSE(fd) < 0) {
> +                virReportSystemError(errno,
> +                                     _("Unable to save lease %s"),
> +                                     res->disks[0].path);
> +                goto error_unlink;
> +            }
> +
> +            if ((rv = sanlock_direct_init(NULL, res, driver->maxHosts, driver->maxHosts, 0)) < 0) {
> +                if (rv <= -200)
> +                    virLockError(VIR_ERR_INTERNAL_ERROR,
> +                                 _("Unable to initialize lease %s: error %d"),
> +                                 res->disks[0].path, rv);
> +                else
> +                    virReportSystemError(-rv,
> +                                         _("Unable to initialize lease %s"),
> +                                         res->disks[0].path);
> +                goto error_unlink;
> +            }
> +            VIR_DEBUG("Lease %s has been initialized", res->disks[0].path);
> +        }
> +    }
> +
>      return 0;
>  
> -error:
> -    VIR_FREE(res);
> +error_unlink:
> +    unlink(res->disks[0].path);
> +    VIR_FORCE_CLOSE(fd);
>      return -1;
>  }
>  
> +
> +static int virLockManagerSanlockAddResource(virLockManagerPtr lock,
> +                                            unsigned int type,
> +                                            const char *name,
> +                                            size_t nparams,
> +                                            virLockManagerParamPtr params,
> +                                            unsigned int flags)
> +{
> +    virLockManagerSanlockPrivatePtr priv = lock->privateData;
> +
> +    virCheckFlags(VIR_LOCK_MANAGER_RESOURCE_READONLY |
> +                  VIR_LOCK_MANAGER_RESOURCE_SHARED, -1);
> +
> +    if (priv->res_count == SANLK_MAX_RESOURCES) {
> +        virLockError(VIR_ERR_INTERNAL_ERROR,
> +                     _("Too many resources %d for object"),
> +                     SANLK_MAX_RESOURCES);
> +        return -1;
> +    }
> +
> +    if (flags & VIR_LOCK_MANAGER_RESOURCE_READONLY) {
> +        virLockError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> +                     _("Readonly leases are not supported"));
> +        return -1;
> +    }
> +    if (flags & VIR_LOCK_MANAGER_RESOURCE_SHARED) {
> +        virLockError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> +                     _("Sharable leases are not supported"));
> +        return -1;
> +    }
> +
> +    switch (type) {
> +    case VIR_LOCK_MANAGER_RESOURCE_TYPE_DISK:
> +        if (driver->autoDiskLease) {
> +            if (virLockManagerSanlockAddDisk(lock, name, nparams, params) < 0)
> +                return -1;
> +
> +            if (virLockManagerSanlockCreateLease(priv->res_args[priv->res_count-1]) < 0)
> +                return -1;
> +        } else {
> +            if (!(flags & (VIR_LOCK_MANAGER_RESOURCE_SHARED |
> +                           VIR_LOCK_MANAGER_RESOURCE_READONLY)))
> +                priv->hasRWDisks = true;
> +            /* Ignore disk resources without error */
> +        }
> +        break;
> +
> +    case VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE:
> +        if (virLockManagerSanlockAddLease(lock, name, nparams, params) < 0)
> +            return -1;
> +        break;
> +
> +    default:
> +        /* Ignore other resources, without error */
> +        break;
> +    }
> +
> +    return 0;
> +}
> +
>  static int virLockManagerSanlockAcquire(virLockManagerPtr lock,
>                                          const char *state,
>                                          unsigned int flags)
> diff --git a/src/locking/sanlock.conf b/src/locking/sanlock.conf
> index 818b529..fb04ea6 100644
> --- a/src/locking/sanlock.conf
> +++ b/src/locking/sanlock.conf
> @@ -1,6 +1,66 @@
> +
> +#
> +# The default sanlock configuration requires the management
> +# application to manually define <lease> elements in the
> +# guest configuration, typically one lease per disk. An
> +# alternative is to enable "auto disk lease" mode. In this
> +# usage, libvirt will automatically create a lockspace and
> +# lease for each fully qualified disk path. This works if
> +# you are able to ensure stable, unique disk paths across
> +# all hosts in a network.
> +#
> +# Uncomment this to enable automatic lease creation.
> +#
> +# NB: the 'host_id' parameter must be set if enabling this
> +#
> +#auto_disk_leases = 1
> +
> +#
> +# The default location in which lockspaces are created when
> +# automatic lease creation is enabled. For each unique disk
> +# path, a file $LEASE_DIR/NNNNNNNNNNNNNN will be created
> +# where 'NNNNNNNNNNNN' is the MD5 checkout of the disk path.
> +#
> +# If this directory is on local storage, it will only protect
> +# against a VM being started twice on the same host, or two
> +# guests on the same host using the same disk path. If the
> +# directory is on NFS, then it can protect against concurrent
> +# usage across all hosts which have the share mounted.
> +#
> +# Recommendation is to just mount this default location as
> +# an NFS volume. Uncomment this, if you would prefer the mount
> +# point to be somewhere else.
> +#
> +#disk_lease_dir = "/var/lib/libvirt/sanlock"
> +
> +#
> +# When automatically creating leases for disks, each host which
> +# can access the filesystem mounted at 'disk_lease_dir' will need
> +# to have a unique host ID assigned. The 'max_hosts' parameter
> +# specifies an upper limit on the number of participating hosts.
> +#
> +# Each additional host requires 1 sector of disk space, usually
> +# 512 bytes. The default is 64, and can be reduced if you don't
> +# have many hosts, or increased if you have more.
> +#
> +#max_hosts = 64
> +
> +#
> +# The unique ID for this host.
> +#
> +# IMPORTANT: *EVERY* host which can access the filesystem mounted
> +# at 'disk_lease_dir' *MUST* be given a different host ID.
> +#
> +# This parameter has no default and must be manually set if
> +# 'auto_disk_leases' is enabled
> +#host_id = 1
> +
>  #
>  # Flag to determine whether we allow starting of guests
>  # which do not have any <lease> elements defined in their
>  # configuration.
>  #
> +# If 'auto_disk_leases' is disabled, this setting defaults
> +# to enabled, otherwise it defaults to disabled.
> +#
>  #require_lease_for_disks = 1
> diff --git a/src/locking/test_libvirt_sanlock.aug b/src/locking/test_libvirt_sanlock.aug
> index 2f1f57a..26a0608 100644
> --- a/src/locking/test_libvirt_sanlock.aug
> +++ b/src/locking/test_libvirt_sanlock.aug
> @@ -1,7 +1,15 @@
>  module Test_libvirt_sanlock =
>  
> -   let conf = "require_lease_for_disks = 1
> +   let conf = "auto_disk_leases = 1
> +disk_lease_dir = \"/var/lib/libvirt/sanlock\"
> +max_hosts = 64
> +host_id = 1
> +require_lease_for_disks = 1
>  "
>  
>     test Libvirt_sanlock.lns get conf =
> +{ "auto_disk_leases" = "1" }
> +{ "disk_lease_dir" = "/var/lib/libvirt/sanlock" }
> +{ "max_hosts" = "64" }
> +{ "host_id" = "1" }
>  { "require_lease_for_disks" = "1" }
> diff --git a/tools/.gitignore b/tools/.gitignore
> index 3e34ea8..a92735d 100644
> --- a/tools/.gitignore
> +++ b/tools/.gitignore
> @@ -1,7 +1,10 @@
>  libvirt-guests.init
>  virt-xml-validate
>  virt-pki-validate
> +virt-sanlock-cleanup
> +virt-sanlock-cleanup.cron
>  *.1
> +*.8
>  Makefile
>  Makefile.in
>  virsh-net-edit.c
> diff --git a/tools/Makefile.am b/tools/Makefile.am
> index 826674a..81e758e 100644
> --- a/tools/Makefile.am
> +++ b/tools/Makefile.am
> @@ -12,6 +12,8 @@ EXTRA_DIST = \
>  	$(ICON_FILES)					\
>  	virt-xml-validate.in				\
>  	virt-pki-validate.in				\
> +	virt-sanlock-cleanup.in				\
> +	virt-sanlock-cleanup.cron.in			\
>  	virsh.pod					\
>  	libvirt-guests.init.sh				\
>  	libvirt-guests.sysconf
> @@ -19,8 +21,17 @@ EXTRA_DIST = \
>  bin_SCRIPTS = virt-xml-validate virt-pki-validate
>  bin_PROGRAMS = virsh
>  
> -dist_man1_MANS = virt-xml-validate.1 virt-pki-validate.1 virsh.1
> +if HAVE_SANLOCK
> +sbin_SCRIPTS = virt-sanlock-cleanup
> +
> +crondir = $(sysconfdir)/cron.daily
> +cron_DATA = virt-sanlock-cleanup.cron
> +endif
>  
> +dist_man1_MANS = virt-xml-validate.1 virt-pki-validate.1 virsh.1
> +if HAVE_SANLOCK
> +dist_man8_MANS = virt-sanlock-cleanup.8
> +endif
>  
>  virt-xml-validate: virt-xml-validate.in Makefile
>  	$(AM_V_GEN)sed -e 's,@SCHEMADIR@,$(pkgdatadir)/schemas,' < $< > $@ \
> @@ -36,6 +47,18 @@ virt-pki-validate: virt-pki-validate.in Makefile
>  virt-pki-validate.1: virt-pki-validate.in
>  	$(AM_V_GEN)$(POD2MAN) $< $(srcdir)/$@
>  
> +virt-sanlock-cleanup: virt-sanlock-cleanup.in Makefile
> +	$(AM_V_GEN)sed -e 's,@SYSCONFDIR@,$(sysconfdir),' \
> +            -e 's,@LOCALSTATEDIR@,$(localstatedir),' < $< > $@ \
> +	    || (rm $@ && exit 1) && chmod +x $@
> +
> +virt-sanlock-cleanup.cron: virt-sanlock-cleanup.cron.in Makefile
> +	$(AM_V_GEN)sed -e 's,@SBINDIR@,$(sbindir),' < $< > $@ \
> +	    || (rm $@ && exit 1) && chmod +x $@
> +
> +virt-sanlock-cleanup.8: virt-sanlock-cleanup.in
> +	$(AM_V_GEN)$(POD2MAN) $< $(srcdir)/$@
> +
>  virsh_SOURCES =							\
>  		console.c console.h				\
>  		virsh.c
> diff --git a/tools/virt-sanlock-cleanup.cron.in b/tools/virt-sanlock-cleanup.cron.in
> new file mode 100644
> index 0000000..14a1773
> --- /dev/null
> +++ b/tools/virt-sanlock-cleanup.cron.in
> @@ -0,0 +1,4 @@
> +#!/bin/sh
> +
> + SBINDIR@/virt-sanlock-cleanup -q
> +exit 0
> diff --git a/tools/virt-sanlock-cleanup.in b/tools/virt-sanlock-cleanup.in
> new file mode 100644
> index 0000000..483b595
> --- /dev/null
> +++ b/tools/virt-sanlock-cleanup.in
> @@ -0,0 +1,91 @@
> +#!/bin/sh
> +
> +# A script to cleanup resource leases auto-created by
> +# the libvirt lock plugin for sanlock
> +
> +verbose=1
> +if test "$1" = "-q" ; then
> +  verbose=0
> +fi
> +
> +LOCKSPACE="__LIBVIRT__DISKS__"
> +
> +LOCKDIR=`augtool print /files SYSCONFDIR@/libvirt/qemu-sanlock.conf/disk_lease_dir`
> +if test $? != 0 -o "x$DIR" = "x" ; then
> +  LOCKDIR="@LOCALSTATEDIR@/lib/libvirt/sanlock"
> +fi
> +
> +function notify() {
> +  if test $verbose = 1 ; then
> +     echo "$@"
> +  fi
> +}
> +
> +cd $LOCKDIR
> +
> +for MD5 in *
> +do
> +  if test $MD5 != '*' -a $MD5 != $LOCKSPACE ; then
> +    RESOURCE="$LOCKSPACE:$MD5:$LOCKDIR/$MD5:0"
> +    notify -n "Cleanup: $RESOURCE "
> +    sanlock client command -r $RESOURCE -c /bin/rm -f "$LOCKDIR/$MD5" 2>/dev/null
> +    if test $? = 0 ; then
> +      notify "PASS"
> +    else
> +      notify "FAIL"
> +    fi
> +  fi
> +done
> +
> +exit 0
> +
> +: <<=cut
> +=pod
> +
> +=head1 NAME
> +
> +  virt-sanlock-cleanup - remove stale sanlock resource lease files
> +
> +=head1 SYNOPSIS
> +
> +  virt-sanlock-cleanup
> +
> +=head1 DESCRIPTION
> +
> +This tool removes any resource lease files created by the sanlock
> +lock manager plugin. The resource lease files only need to exist
> +on disks when a guest using the resource is active. This script
> +reclaims the disk space used by resources which are not currently
> +active.
> +
> +=head1 EXIT STATUS
> +
> +Upon successful cleanup, an exit status of 0 will be set. Upon
> +failure a non-zero status will be set.
> +
> +=head1 AUTHOR
> +
> +Daniel Berrange
> +
> +=head1 BUGS
> +
> +Report any bugs discovered to the libvirt community via the
> +mailing list C<http://libvirt.org/contact.html> or bug tracker C<http://libvirt.org/bugs.html>.
> +Alternatively report bugs to your software distributor / vendor.
> +
> +=head1 COPYRIGHT
> +
> +Copyright (C) 2011 Red Hat, Inc.
> +
> +=head1 LICENSE
> +
> +virt-sanlock-cleanup is distributed under the terms of the GNU GPL v2+.
> +This is free software; see the source for copying conditions. There
> +is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR
> +PURPOSE
> +
> +=head1 SEE ALSO
> +
> +C<virsh(1)>, online instructions C<http://libvirt.org/locksanlock.html>
> +
> +=cut

  ACK, let's push this that allows most users to benefits from the
  locking if they don't have a preexisting infrastructure,

Daniel

-- 
Daniel Veillard      | libxml Gnome XML XSLT toolkit  http://xmlsoft.org/
daniel veillard com  | Rpmfind RPM search engine http://rpmfind.net/
http://veillard.com/ | virtualization library  http://libvirt.org/


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]