[libvirt] [PATCH] storage backend: Add RBD (RADOS Block Device) support
Wido den Hollander
wido at widodh.nl
Wed Apr 11 17:47:10 UTC 2012
Hi,
On 03/30/2012 11:04 AM, Wido den Hollander wrote:
> This patch adds support for a new storage backend with RBD support.
>
> RBD is the RADOS Block Device and is part of the Ceph distributed storage system.
>
> It comes in two flavours: Qemu-RBD and Kernel RBD, this storage backend only supports
> Qemu-RBD, thus limiting the use of this storage driver to Qemu only.
>
> To function this backend relies on librbd and librados being present on the local system.
>
> The backend also supports Cephx authentication for safe authentication with the Ceph cluster.
>
> For storing credentials it uses the build-in secret mechanism of libvirt.
Has anyone found the time yet to review this one?
Thanks,
Wido
>
> Signed-off-by: Wido den Hollander<wido at widodh.nl>
> ---
> configure.ac | 20 ++
> include/libvirt/libvirt.h.in | 1 +
> src/Makefile.am | 9 +
> src/conf/storage_conf.c | 197 ++++++++++---
> src/conf/storage_conf.h | 16 +
> src/storage/storage_backend.c | 6 +
> src/storage/storage_backend_rbd.c | 465 ++++++++++++++++++++++++++++++
> src/storage/storage_backend_rbd.h | 30 ++
> tests/storagepoolxml2xmlin/pool-rbd.xml | 11 +
> tests/storagepoolxml2xmlout/pool-rbd.xml | 15 +
> tools/virsh.c | 7 +
> 11 files changed, 734 insertions(+), 43 deletions(-)
> create mode 100644 src/storage/storage_backend_rbd.c
> create mode 100644 src/storage/storage_backend_rbd.h
> create mode 100644 tests/storagepoolxml2xmlin/pool-rbd.xml
> create mode 100644 tests/storagepoolxml2xmlout/pool-rbd.xml
>
> diff --git a/configure.ac b/configure.ac
> index 32cc8d0..b693e5b 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -1743,6 +1743,8 @@ AC_ARG_WITH([storage-mpath],
> AC_HELP_STRING([--with-storage-mpath], [with mpath backend for the storage driver @<:@default=check@:>@]),[],[with_storage_mpath=check])
> AC_ARG_WITH([storage-disk],
> AC_HELP_STRING([--with-storage-disk], [with GPartd Disk backend for the storage driver @<:@default=check@:>@]),[],[with_storage_disk=check])
> +AC_ARG_WITH([storage-rbd],
> + AC_HELP_STRING([--with-storage-rbd], [with RADOS Block Device backend for the storage driver @<:@default=check@:>@]),[],[with_storage_rbd=check])
>
> if test "$with_libvirtd" = "no"; then
> with_storage_dir=no
> @@ -1752,6 +1754,7 @@ if test "$with_libvirtd" = "no"; then
> with_storage_scsi=no
> with_storage_mpath=no
> with_storage_disk=no
> + with_storage_rbd=no
> fi
> if test "$with_storage_dir" = "yes" ; then
> AC_DEFINE_UNQUOTED([WITH_STORAGE_DIR], 1, [whether directory backend for storage driver is enabled])
> @@ -1910,6 +1913,22 @@ if test "$with_storage_mpath" = "check"; then
> fi
> AM_CONDITIONAL([WITH_STORAGE_MPATH], [test "$with_storage_mpath" = "yes"])
>
> +if test "$with_storage_rbd" = "yes" || test "$with_storage_rbd" = "check"; then
> + AC_CHECK_HEADER([rbd/librbd.h], [LIBRBD_FOUND=yes; break;])
> +
> + LIBRBD_LIBS="-lrbd -lrados -lcrypto"
> +
> + if test "$LIBRBD_FOUND" = "yes"; then
> + with_storage_rbd=yes
> + LIBS="$LIBS $LIBRBD_LIBS"
> + else
> + with_storage_rbd=no
> + fi
> +
> + AC_DEFINE_UNQUOTED([WITH_STORAGE_RBD], 1, [wether RBD backend for storage driver is enabled])
> +fi
> +AM_CONDITIONAL([WITH_STORAGE_RBD], [test "$with_storage_rbd" = "yes"])
> +
> LIBPARTED_CFLAGS=
> LIBPARTED_LIBS=
> if test "$with_storage_disk" = "yes" ||
> @@ -2673,6 +2692,7 @@ AC_MSG_NOTICE([ iSCSI: $with_storage_iscsi])
> AC_MSG_NOTICE([ SCSI: $with_storage_scsi])
> AC_MSG_NOTICE([ mpath: $with_storage_mpath])
> AC_MSG_NOTICE([ Disk: $with_storage_disk])
> +AC_MSG_NOTICE([ RBD: $with_storage_rbd])
> AC_MSG_NOTICE([])
> AC_MSG_NOTICE([Security Drivers])
> AC_MSG_NOTICE([])
> diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in
> index 499dcd4..ee1d5ec 100644
> --- a/include/libvirt/libvirt.h.in
> +++ b/include/libvirt/libvirt.h.in
> @@ -2324,6 +2324,7 @@ typedef enum {
> VIR_STORAGE_VOL_FILE = 0, /* Regular file based volumes */
> VIR_STORAGE_VOL_BLOCK = 1, /* Block based volumes */
> VIR_STORAGE_VOL_DIR = 2, /* Directory-passthrough based volume */
> + VIR_STORAGE_VOL_NETWORK = 3, /* Network volumes like RBD (RADOS Block Device) */
>
> #ifdef VIR_ENUM_SENTINELS
> VIR_STORAGE_VOL_LAST
> diff --git a/src/Makefile.am b/src/Makefile.am
> index a2aae9d..e4457c3 100644
> --- a/src/Makefile.am
> +++ b/src/Makefile.am
> @@ -498,6 +498,9 @@ STORAGE_DRIVER_MPATH_SOURCES = \
> STORAGE_DRIVER_DISK_SOURCES = \
> storage/storage_backend_disk.h storage/storage_backend_disk.c
>
> +STORAGE_DRIVER_RBD_SOURCES = \
> + storage/storage_backend_rbd.h storage/storage_backend_rbd.c
> +
> STORAGE_HELPER_DISK_SOURCES = \
> storage/parthelper.c
>
> @@ -1040,6 +1043,11 @@ if WITH_STORAGE_DISK
> libvirt_driver_storage_la_SOURCES += $(STORAGE_DRIVER_DISK_SOURCES)
> endif
>
> +if WITH_STORAGE_RBD
> +libvirt_driver_storage_la_SOURCES += $(STORAGE_DRIVER_RBD_SOURCES)
> +libvirt_la_LIBADD += $(LIBRBD_LIBS)
> +endif
> +
> if WITH_NODE_DEVICES
> # Needed to keep automake quiet about conditionals
> if WITH_DRIVER_MODULES
> @@ -1139,6 +1147,7 @@ EXTRA_DIST += \
> $(STORAGE_DRIVER_SCSI_SOURCES) \
> $(STORAGE_DRIVER_MPATH_SOURCES) \
> $(STORAGE_DRIVER_DISK_SOURCES) \
> + $(STORAGE_DRIVER_RBD_SOURCES) \
> $(NODE_DEVICE_DRIVER_SOURCES) \
> $(NODE_DEVICE_DRIVER_HAL_SOURCES) \
> $(NODE_DEVICE_DRIVER_UDEV_SOURCES) \
> diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
> index bdf6218..2a0b5eb 100644
> --- a/src/conf/storage_conf.c
> +++ b/src/conf/storage_conf.c
> @@ -52,7 +52,7 @@ VIR_ENUM_IMPL(virStoragePool,
> VIR_STORAGE_POOL_LAST,
> "dir", "fs", "netfs",
> "logical", "disk", "iscsi",
> - "scsi", "mpath")
> + "scsi", "mpath", "rbd")
>
> VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
> VIR_STORAGE_POOL_FS_LAST,
> @@ -110,6 +110,7 @@ enum {
> VIR_STORAGE_POOL_SOURCE_ADAPTER = (1<<3),
> VIR_STORAGE_POOL_SOURCE_NAME = (1<<4),
> VIR_STORAGE_POOL_SOURCE_INITIATOR_IQN = (1<<5),
> + VIR_STORAGE_POOL_SOURCE_NETWORK = (1<<6),
> };
>
>
> @@ -194,6 +195,15 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
> .formatToString = virStoragePoolFormatDiskTypeToString,
> }
> },
> + { .poolType = VIR_STORAGE_POOL_RBD,
> + .poolOptions = {
> + .flags = (VIR_STORAGE_POOL_SOURCE_NETWORK |
> + VIR_STORAGE_POOL_SOURCE_NAME),
> + },
> + .volOptions = {
> + .formatToString = virStoragePoolFormatDiskTypeToString,
> + }
> + },
> { .poolType = VIR_STORAGE_POOL_MPATH,
> .volOptions = {
> .formatToString = virStoragePoolFormatDiskTypeToString,
> @@ -277,6 +287,11 @@ virStoragePoolSourceClear(virStoragePoolSourcePtr source)
> return;
>
> VIR_FREE(source->host.name);
> + for (i = 0 ; i< source->nhost ; i++) {
> + VIR_FREE(source->hosts[i].name);
> + }
> + VIR_FREE(source->hosts);
> +
> for (i = 0 ; i< source->ndevice ; i++) {
> VIR_FREE(source->devices[i].freeExtents);
> VIR_FREE(source->devices[i].path);
> @@ -293,6 +308,12 @@ virStoragePoolSourceClear(virStoragePoolSourcePtr source)
> VIR_FREE(source->auth.chap.login);
> VIR_FREE(source->auth.chap.passwd);
> }
> +
> + if (source->authType == VIR_STORAGE_POOL_AUTH_CEPHX) {
> + VIR_FREE(source->auth.cephx.username);
> + VIR_FREE(source->auth.cephx.secret.uuid);
> + VIR_FREE(source->auth.cephx.secret.usage);
> + }
> }
>
> void
> @@ -395,6 +416,27 @@ virStoragePoolDefParseAuthChap(xmlXPathContextPtr ctxt,
> }
>
> static int
> +virStoragePoolDefParseAuthCephx(xmlXPathContextPtr ctxt,
> + virStoragePoolAuthCephxPtr auth) {
> + auth->username = virXPathString("string(./auth/@username)", ctxt);
> + if (auth->username == NULL) {
> + virStorageReportError(VIR_ERR_XML_ERROR,
> + "%s", _("missing auth username attribute"));
> + return -1;
> + }
> +
> + auth->secret.uuid = virXPathString("string(./auth/secret/@uuid)", ctxt);
> + auth->secret.usage = virXPathString("string(./auth/secret/@usage)", ctxt);
> + if (auth->secret.uuid == NULL&& auth->secret.usage == NULL) {
> + virStorageReportError(VIR_ERR_XML_ERROR,
> + "%s", _("missing auth secret uuid or usage attribute"));
> + return -1;
> + }
> +
> + return 0;
> +}
> +
> +static int
> virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
> virStoragePoolSourcePtr source,
> int pool_type,
> @@ -414,6 +456,12 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
> }
>
> source->name = virXPathString("string(./name)", ctxt);
> + if (pool_type == VIR_STORAGE_POOL_RBD&& source->name == NULL) {
> + virStorageReportError(VIR_ERR_XML_ERROR,
> + _("%s"), "missing mandatory 'name' field for RBD pool name");
> + VIR_FREE(source->name);
> + goto cleanup;
> + }
>
> if (options->formatFromString) {
> char *format = virXPathString("string(./format/@type)", ctxt);
> @@ -431,17 +479,39 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
> VIR_FREE(format);
> }
>
> - source->host.name = virXPathString("string(./host/@name)", ctxt);
> - port = virXPathString("string(./host/@port)", ctxt);
> - if (port) {
> - if (virStrToLong_i(port, NULL, 10,&source->host.port)< 0) {
> - virStorageReportError(VIR_ERR_XML_ERROR,
> - _("Invalid port number: %s"),
> - port);
> + source->nhost = virXPathNodeSet("./host", ctxt,&nodeset);
> +
> + if (source->nhost) {
> + if (VIR_ALLOC_N(source->hosts, source->nhost)< 0) {
> + virReportOOMError();
> goto cleanup;
> }
> - }
>
> + for (i = 0 ; i< source->nhost ; i++) {
> + char *name = virXMLPropString(nodeset[i], "name");
> + if (name == NULL) {
> + virStorageReportError(VIR_ERR_XML_ERROR,
> + "%s", _("missing storage pool host name"));
> + goto cleanup;
> + }
> + source->hosts[i].name = name;
> + if(i == 0&& source->nhost == 1)
> + source->host.name = name;
> +
> + port = virXMLPropString(nodeset[i], "port");
> + if (port) {
> + if (virStrToLong_i(port, NULL, 10,&source->hosts[i].port)< 0) {
> + virStorageReportError(VIR_ERR_XML_ERROR,
> + _("Invalid port number: %s"),
> + port);
> + goto cleanup;
> + } else {
> + if (i == 0&& source->nhost == 1)
> + virStrToLong_i(port, NULL, 10,&source->host.port);
> + }
> + }
> + }
> + }
>
> source->initiator.iqn = virXPathString("string(./initiator/iqn/@name)", ctxt);
>
> @@ -478,6 +548,8 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
> } else {
> if (STREQ(authType, "chap")) {
> source->authType = VIR_STORAGE_POOL_AUTH_CHAP;
> + } else if (STREQ(authType, "ceph")) {
> + source->authType = VIR_STORAGE_POOL_AUTH_CEPHX;
> } else {
> virStorageReportError(VIR_ERR_XML_ERROR,
> _("unknown auth type '%s'"),
> @@ -491,6 +563,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
> goto cleanup;
> }
>
> + if (source->authType == VIR_STORAGE_POOL_AUTH_CEPHX) {
> + if (virStoragePoolDefParseAuthCephx(ctxt,&source->auth.cephx)< 0)
> + goto cleanup;
> + }
> +
> source->vendor = virXPathString("string(./vendor/@name)", ctxt);
> source->product = virXPathString("string(./product/@name)", ctxt);
>
> @@ -682,6 +759,15 @@ virStoragePoolDefParseXML(xmlXPathContextPtr ctxt) {
> }
> }
>
> + if (options->flags& VIR_STORAGE_POOL_SOURCE_NETWORK) {
> + if (!ret->source.host.name&& ret->source.nhost< 1) {
> + virStorageReportError(VIR_ERR_XML_ERROR,
> + "%s",
> + _("missing storage pool source network host name"));
> + goto cleanup;
> + }
> + }
> +
> if (options->flags& VIR_STORAGE_POOL_SOURCE_DIR) {
> if (!ret->source.dir) {
> virStorageReportError(VIR_ERR_XML_ERROR,
> @@ -717,20 +803,22 @@ virStoragePoolDefParseXML(xmlXPathContextPtr ctxt) {
> }
> }
>
> - if ((tmppath = virXPathString("string(./target/path)", ctxt)) == NULL) {
> - virStorageReportError(VIR_ERR_XML_ERROR,
> - "%s", _("missing storage pool target path"));
> - goto cleanup;
> - }
> - ret->target.path = virFileSanitizePath(tmppath);
> - VIR_FREE(tmppath);
> - if (!ret->target.path)
> - goto cleanup;
> -
> + /* When we are working with a virtual disk we can skip the target path and permissions */
> + if (!(options->flags& VIR_STORAGE_POOL_SOURCE_NETWORK)) {
> + if ((tmppath = virXPathString("string(./target/path)", ctxt)) == NULL) {
> + virStorageReportError(VIR_ERR_XML_ERROR,
> + "%s", _("missing storage pool target path"));
> + goto cleanup;
> + }
> + ret->target.path = virFileSanitizePath(tmppath);
> + VIR_FREE(tmppath);
> + if (!ret->target.path)
> + goto cleanup;
>
> - if (virStorageDefParsePerms(ctxt,&ret->target.perms,
> - "./target/permissions", 0700)< 0)
> - goto cleanup;
> + if (virStorageDefParsePerms(ctxt,&ret->target.perms,
> + "./target/permissions", 0700)< 0)
> + goto cleanup;
> + }
>
> return ret;
>
> @@ -800,12 +888,15 @@ virStoragePoolSourceFormat(virBufferPtr buf,
> int i, j;
>
> virBufferAddLit(buf,"<source>\n");
> - if ((options->flags& VIR_STORAGE_POOL_SOURCE_HOST)&&
> - src->host.name) {
> - virBufferAsprintf(buf, "<host name='%s'", src->host.name);
> - if (src->host.port)
> - virBufferAsprintf(buf, " port='%d'", src->host.port);
> - virBufferAddLit(buf, "/>\n");
> + if ((options->flags& VIR_STORAGE_POOL_SOURCE_HOST ||
> + options->flags& VIR_STORAGE_POOL_SOURCE_NETWORK)&&
> + src->nhost) {
> + for (i = 0; i< src->nhost; i++) {
> + virBufferAsprintf(buf, "<host name='%s'", src->hosts[i].name);
> + if (src->hosts[i].port)
> + virBufferAsprintf(buf, " port='%d'", src->hosts[i].port);
> + virBufferAddLit(buf, "/>\n");
> + }
> }
>
> if ((options->flags& VIR_STORAGE_POOL_SOURCE_DEVICE)&&
> @@ -860,6 +951,23 @@ virStoragePoolSourceFormat(virBufferPtr buf,
> src->auth.chap.login,
> src->auth.chap.passwd);
>
> + if (src->authType == VIR_STORAGE_POOL_AUTH_CEPHX) {
> + virBufferAsprintf(buf,"<auth username='%s' type='ceph'>\n",
> + src->auth.cephx.username);
> +
> + virBufferAsprintf(buf," %s", "<secret");
> + if (src->auth.cephx.secret.uuid != NULL) {
> + virBufferAsprintf(buf," uuid='%s'", src->auth.cephx.secret.uuid);
> + }
> +
> + if (src->auth.cephx.secret.usage != NULL) {
> + virBufferAsprintf(buf," usage='%s'", src->auth.cephx.secret.usage);
> + }
> + virBufferAsprintf(buf,"%s", "/>\n");
> +
> + virBufferAsprintf(buf," %s", "</auth>\n");
> + }
> +
> if (src->vendor != NULL) {
> virBufferEscapeString(buf,"<vendor name='%s'/>\n", src->vendor);
> }
> @@ -907,25 +1015,28 @@ virStoragePoolDefFormat(virStoragePoolDefPtr def) {
> if (virStoragePoolSourceFormat(&buf, options,&def->source)< 0)
> goto cleanup;
>
> - virBufferAddLit(&buf,"<target>\n");
> + /* RBD devices are no local block devs nor files, so it doesn't have a target */
> + if (def->type != VIR_STORAGE_POOL_RBD) {
> + virBufferAddLit(&buf,"<target>\n");
>
> - if (def->target.path)
> - virBufferAsprintf(&buf,"<path>%s</path>\n", def->target.path);
> + if (def->target.path)
> + virBufferAsprintf(&buf,"<path>%s</path>\n", def->target.path);
>
> - virBufferAddLit(&buf,"<permissions>\n");
> - virBufferAsprintf(&buf,"<mode>0%o</mode>\n",
> - def->target.perms.mode);
> - virBufferAsprintf(&buf,"<owner>%d</owner>\n",
> - def->target.perms.uid);
> - virBufferAsprintf(&buf,"<group>%d</group>\n",
> - def->target.perms.gid);
> + virBufferAddLit(&buf,"<permissions>\n");
> + virBufferAsprintf(&buf,"<mode>0%o</mode>\n",
> + def->target.perms.mode);
> + virBufferAsprintf(&buf,"<owner>%d</owner>\n",
> + def->target.perms.uid);
> + virBufferAsprintf(&buf,"<group>%d</group>\n",
> + def->target.perms.gid);
>
> - if (def->target.perms.label)
> - virBufferAsprintf(&buf,"<label>%s</label>\n",
> - def->target.perms.label);
> + if (def->target.perms.label)
> + virBufferAsprintf(&buf,"<label>%s</label>\n",
> + def->target.perms.label);
>
> - virBufferAddLit(&buf,"</permissions>\n");
> - virBufferAddLit(&buf,"</target>\n");
> + virBufferAddLit(&buf,"</permissions>\n");
> + virBufferAddLit(&buf,"</target>\n");
> + }
> virBufferAddLit(&buf,"</pool>\n");
>
> if (virBufferError(&buf))
> diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
> index 1ef9295..6b91ca5 100644
> --- a/src/conf/storage_conf.h
> +++ b/src/conf/storage_conf.h
> @@ -120,6 +120,7 @@ enum virStoragePoolType {
> VIR_STORAGE_POOL_ISCSI, /* iSCSI targets */
> VIR_STORAGE_POOL_SCSI, /* SCSI HBA */
> VIR_STORAGE_POOL_MPATH, /* Multipath devices */
> + VIR_STORAGE_POOL_RBD, /* RADOS Block Device */
>
> VIR_STORAGE_POOL_LAST,
> };
> @@ -137,6 +138,7 @@ enum virStoragePoolDeviceType {
> enum virStoragePoolAuthType {
> VIR_STORAGE_POOL_AUTH_NONE,
> VIR_STORAGE_POOL_AUTH_CHAP,
> + VIR_STORAGE_POOL_AUTH_CEPHX,
> };
>
> typedef struct _virStoragePoolAuthChap virStoragePoolAuthChap;
> @@ -146,6 +148,15 @@ struct _virStoragePoolAuthChap {
> char *passwd;
> };
>
> +typedef struct _virStoragePoolAuthCephx virStoragePoolAuthCephx;
> +typedef virStoragePoolAuthCephx *virStoragePoolAuthCephxPtr;
> +struct _virStoragePoolAuthCephx {
> + char *username;
> + struct {
> + char *uuid;
> + char *usage;
> + } secret;
> +};
>
> /*
> * For remote pools, info on how to reach the host
> @@ -215,6 +226,10 @@ struct _virStoragePoolSource {
> /* An optional host */
> virStoragePoolSourceHost host;
>
> + /* Or multiple hosts */
> + int nhost;
> + virStoragePoolSourceHostPtr hosts;
> +
> /* And either one or more devices ... */
> int ndevice;
> virStoragePoolSourceDevicePtr devices;
> @@ -234,6 +249,7 @@ struct _virStoragePoolSource {
> int authType; /* virStoragePoolAuthType */
> union {
> virStoragePoolAuthChap chap;
> + virStoragePoolAuthCephx cephx;
> } auth;
>
> /* Vendor of the source */
> diff --git a/src/storage/storage_backend.c b/src/storage/storage_backend.c
> index caac2f8..e2e9b51 100644
> --- a/src/storage/storage_backend.c
> +++ b/src/storage/storage_backend.c
> @@ -77,6 +77,9 @@
> #if WITH_STORAGE_DIR
> # include "storage_backend_fs.h"
> #endif
> +#if WITH_STORAGE_RBD
> +# include "storage_backend_rbd.h"
> +#endif
>
> #define VIR_FROM_THIS VIR_FROM_STORAGE
>
> @@ -103,6 +106,9 @@ static virStorageBackendPtr backends[] = {
> #if WITH_STORAGE_DISK
> &virStorageBackendDisk,
> #endif
> +#if WITH_STORAGE_RBD
> +&virStorageBackendRBD,
> +#endif
> NULL
> };
>
> diff --git a/src/storage/storage_backend_rbd.c b/src/storage/storage_backend_rbd.c
> new file mode 100644
> index 0000000..056059a
> --- /dev/null
> +++ b/src/storage/storage_backend_rbd.c
> @@ -0,0 +1,465 @@
> +/*
> + * storage_backend_rbd.c: storage backend for RBD (RADOS Block Device) handling
> + *
> + * Copyright (C) 2012 Wido den Hollander
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library; if not, write to the Free Software
> + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
> + *
> + * Author: Wido den Hollander<wido at widodh.nl>
> + */
> +
> +#include<config.h>
> +
> +#include "virterror_internal.h"
> +#include "storage_backend_rbd.h"
> +#include "storage_conf.h"
> +#include "util.h"
> +#include "memory.h"
> +#include "logging.h"
> +#include "base64.h"
> +#include "rados/librados.h"
> +#include "rbd/librbd.h"
> +
> +#define VIR_FROM_THIS VIR_FROM_STORAGE
> +
> +struct _virStorageBackendRBDState {
> + rados_t cluster;
> + rados_ioctx_t ioctx;
> + time_t starttime;
> +};
> +
> +typedef struct _virStorageBackendRBDState virStorageBackendRBDState;
> +typedef virStorageBackendRBDState virStorageBackendRBDStatePtr;
> +
> +static int virStorageBackendRBDOpenRADOSConn(virStorageBackendRBDStatePtr *ptr,
> + virConnectPtr conn,
> + virStoragePoolObjPtr pool)
> +{
> + int ret = -1;
> + unsigned char *secret_value;
> + size_t secret_value_size;
> + char *rados_key;
> + virBuffer mon_host = VIR_BUFFER_INITIALIZER;
> + virSecretPtr secret = NULL;
> +
> + VIR_DEBUG("Found Cephx username: %s",
> + pool->def->source.auth.cephx.username);
> +
> + if (pool->def->source.auth.cephx.username != NULL) {
> + VIR_DEBUG("Using cephx authorization");
> + if (rados_create(&ptr->cluster,
> + pool->def->source.auth.cephx.username)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to initialize RADOS"));
> + goto cleanup;
> + }
> +
> + if (pool->def->source.auth.cephx.secret.uuid != NULL) {
> + VIR_DEBUG("Looking up secret by UUID: %s",
> + pool->def->source.auth.cephx.secret.uuid);
> + secret = virSecretLookupByUUIDString(conn,
> + pool->def->source.auth.cephx.secret.uuid);
> + }
> +
> + if (pool->def->source.auth.cephx.secret.usage != NULL) {
> + VIR_DEBUG("Looking up secret by usage: %s",
> + pool->def->source.auth.cephx.secret.usage);
> + secret = virSecretLookupByUsage(conn, VIR_SECRET_USAGE_TYPE_CEPH,
> + pool->def->source.auth.cephx.secret.usage);
> + }
> +
> + if (secret == NULL) {
> + virStorageReportError(VIR_ERR_NO_SECRET,
> + _("failed to find the secret"));
> + goto cleanup;
> + }
> +
> + secret_value = virSecretGetValue(secret,&secret_value_size, 0);
> + base64_encode_alloc((char *)secret_value,
> + secret_value_size,&rados_key);
> + memset(secret_value, 0, secret_value_size);
> +
> + if (rados_key == NULL) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to decode the RADOS key"));
> + goto cleanup;
> + }
> +
> + VIR_DEBUG("Found cephx key: %s", rados_key);
> + if (rados_conf_set(ptr->cluster, "key", rados_key)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to set RADOS option: %s"),
> + "rados_key");
> + goto cleanup;
> + }
> +
> + memset(rados_key, 0, strlen(rados_key));
> +
> + if (rados_conf_set(ptr->cluster, "auth_supported", "cephx")< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to set RADOS option: %s"),
> + "auth_supported");
> + goto cleanup;
> + }
> + } else {
> + VIR_DEBUG("Not using cephx authorization");
> + if (rados_conf_set(ptr->cluster, "auth_supported", "none")< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to set RADOS option: %s"),
> + "auth_supported");
> + goto cleanup;
> + }
> + if (rados_create(&ptr->cluster, NULL)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to create the RADOS cluster"));
> + goto cleanup;
> + }
> + }
> +
> + VIR_DEBUG("Found %d RADOS cluster monitors in the pool configuration",
> + pool->def->source.nhost);
> +
> + int i;
> + for (i = 0; i< pool->def->source.nhost; i++) {
> + if (pool->def->source.hosts[i].name != NULL&&
> + !pool->def->source.hosts[i].port) {
> + virBufferAsprintf(&mon_host, "%s:6789,",
> + pool->def->source.hosts[i].name);
> + } else if (pool->def->source.hosts[i].name != NULL&&
> + pool->def->source.hosts[i].port) {
> + virBufferAsprintf(&mon_host, "%s:%d,",
> + pool->def->source.hosts[i].name,
> + pool->def->source.hosts[i].port);
> + } else {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("received malformed monitor, check the XML definition"));
> + }
> + }
> +
> + char *mon_buff = virBufferContentAndReset(&mon_host);
> + VIR_DEBUG("RADOS mon_host has been set to: %s", mon_buff);
> + if (rados_conf_set(ptr->cluster, "mon_host", mon_buff)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to set RADOS option: %s"),
> + "mon_host");
> + goto cleanup;
> + }
> +
> + ptr->starttime = time(0);
> + if (rados_connect(ptr->cluster)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to connect to the RADOS monitor on: %s"),
> + virBufferContentAndReset(&mon_host));
> + goto cleanup;
> + }
> +
> + ret = 0;
> +
> +cleanup:
> + VIR_FREE(secret_value);
> + VIR_FREE(rados_key);
> + virSecretFree(secret);
> + virBufferFreeAndReset(&mon_host);
> + return ret;
> +}
> +
> +static int virStorageBackendRBDCloseRADOSConn(virStorageBackendRBDStatePtr ptr)
> +{
> + int ret = 0;
> +
> + if (ptr.ioctx != NULL) {
> + VIR_DEBUG("Closing RADOS IoCTX");
> + rados_ioctx_destroy(ptr.ioctx);
> + ret = -1;
> + }
> +
> + if (ptr.cluster != NULL) {
> + VIR_DEBUG("Closing RADOS connection");
> + rados_shutdown(ptr.cluster);
> + ret = -2;
> + }
> +
> + time_t runtime = time(0) - ptr.starttime;
> + VIR_DEBUG("RADOS connection existed for %ld seconds", runtime);
> +
> + return ret;
> +}
> +
> +static int volStorageBackendRBDRefreshVolInfo(virStorageVolDefPtr vol,
> + virStoragePoolObjPtr pool,
> + virStorageBackendRBDStatePtr ptr)
> +{
> + int ret = -1;
> + rbd_image_t image;
> + if (rbd_open(ptr.ioctx, vol->name,&image, NULL)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to open the RBD image '%s'"),
> + vol->name);
> + goto cleanup;
> + }
> +
> + rbd_image_info_t info;
> + if (rbd_stat(image,&info, sizeof(info))< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> + _("failed to stat the RBD image"));
> + goto cleanup;
> + }
> +
> + VIR_DEBUG("Refreshed RBD image %s/%s (size: %llu obj_size: %llu num_objs: %llu)",
> + pool->def->source.name, vol->name, (unsigned long long)info.size,
> + (unsigned long long)info.obj_size,
> + (unsigned long long)info.num_objs);
> +
> + vol->capacity = info.size;
> + vol->allocation = info.obj_size * info.num_objs;
> + vol->type = VIR_STORAGE_VOL_NETWORK;
> +
> + VIR_FREE(vol->target.path);
> + if (virAsprintf(&vol->target.path, "rbd:%s/%s",
> + pool->def->source.name,
> + vol->name) == -1) {
> + virReportOOMError();
> + goto cleanup;
> + }
> +
> + VIR_FREE(vol->key);
> + if (virAsprintf(&vol->key, "%s/%s",
> + pool->def->source.name,
> + vol->name) == -1) {
> + virReportOOMError();
> + goto cleanup;
> + }
> +
> + ret = 0;
> +
> +cleanup:
> + rbd_close(image);
> + return ret;
> +}
> +
> +static int virStorageBackendRBDRefreshPool(virConnectPtr conn ATTRIBUTE_UNUSED,
> + virStoragePoolObjPtr pool)
> +{
> + size_t max_size = 1024;
> + int ret = -1;
> + virStorageBackendRBDStatePtr ptr;
> + ptr.cluster = NULL;
> + ptr.ioctx = NULL;
> +
> + if (virStorageBackendRBDOpenRADOSConn(&ptr, conn, pool)< 0) {
> + goto cleanup;
> + }
> +
> + if (rados_ioctx_create(ptr.cluster,
> + pool->def->source.name,&ptr.ioctx)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
> + pool->def->source.name);
> + goto cleanup;
> + }
> +
> + struct rados_cluster_stat_t stat;
> + if (rados_cluster_stat(ptr.cluster,&stat)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> + _("failed to stat the RADOS cluster"));
> + goto cleanup;
> + }
> +
> + struct rados_pool_stat_t poolstat;
> + if (rados_ioctx_pool_stat(ptr.ioctx,&poolstat)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to stat the RADOS pool '%s'"),
> + pool->def->source.name);
> + goto cleanup;
> + }
> +
> + pool->def->capacity = stat.kb * 1024;
> + pool->def->available = stat.kb_avail * 1024;
> + pool->def->allocation = poolstat.num_bytes;
> +
> + int num_images, i;
> + char *names, *name = NULL;
> +
> + if (VIR_ALLOC_N(names, 1024)< 0)
> + goto cleanup;
> +
> + int len = rbd_list(ptr.ioctx, names,&max_size);
> +
> + for (i = 0, num_images = 0, name = names; name< names + len; i++) {
> +
> + if (VIR_REALLOC_N(pool->volumes.objs, pool->volumes.count + 1)< 0) {
> + virStoragePoolObjClearVols(pool);
> + virReportOOMError();
> + goto cleanup;
> + }
> +
> + virStorageVolDefPtr vol;
> + if (VIR_ALLOC(vol)< 0)
> + goto cleanup;
> +
> + vol->name = strdup(name);
> + if (vol->name == NULL)
> + goto cleanup;
> + name += strlen(name) + 1;
> +
> + if (volStorageBackendRBDRefreshVolInfo(vol, pool, ptr)< 0)
> + goto cleanup;
> +
> + pool->volumes.objs[pool->volumes.count++] = vol;
> + }
> +
> + VIR_DEBUG("Refreshed RBD pool %s (kb: %llu kb_avail: %llu num_bytes: %llu num_images: %d)",
> + pool->def->source.name, (unsigned long long)stat.kb,
> + (unsigned long long)stat.kb_avail,
> + (unsigned long long)poolstat.num_bytes, pool->volumes.count);
> +
> + ret = 0;
> +
> +cleanup:
> + VIR_FREE(names);
> + virStorageBackendRBDCloseRADOSConn(ptr);
> + return ret;
> +}
> +
> +static int virStorageBackendRBDDeleteVol(virConnectPtr conn,
> + virStoragePoolObjPtr pool,
> + virStorageVolDefPtr vol,
> + unsigned int flags)
> +{
> + int ret = -1;
> + virStorageBackendRBDStatePtr ptr;
> + ptr.cluster = NULL;
> + ptr.ioctx = NULL;
> +
> + VIR_DEBUG("Removing RBD image %s/%s", pool->def->source.name, vol->name);
> +
> + if (virStorageBackendRBDOpenRADOSConn(&ptr, conn, pool)< 0) {
> + goto cleanup;
> + }
> +
> + if (rados_ioctx_create(ptr.cluster,
> + pool->def->source.name,&ptr.ioctx)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
> + pool->def->source.name);
> + goto cleanup;
> + }
> +
> + if (rbd_remove(ptr.ioctx, vol->name)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to remove volume '%s/%s'"),
> + pool->def->source.name,
> + vol->name);
> + goto cleanup;
> + }
> +
> + ret = 0;
> +
> +cleanup:
> + virStorageBackendRBDCloseRADOSConn(ptr);
> + return ret;
> +}
> +
> +static int virStorageBackendRBDCreateVol(virConnectPtr conn,
> + virStoragePoolObjPtr pool,
> + virStorageVolDefPtr vol)
> +{
> + virStorageBackendRBDStatePtr ptr;
> + ptr.cluster = NULL;
> + ptr.ioctx = NULL;
> + int order = 0;
> + int ret = -1;
> +
> + VIR_DEBUG("Creating RBD image %s/%s with size %llu",
> + pool->def->source.name,
> + vol->name, vol->capacity);
> +
> + if (virStorageBackendRBDOpenRADOSConn(&ptr, conn, pool)< 0) {
> + goto cleanup;
> + }
> +
> + if (rados_ioctx_create(ptr.cluster,
> + pool->def->source.name,&ptr.ioctx)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
> + pool->def->source.name);
> + goto cleanup;
> + }
> +
> + if (vol->target.encryption != NULL) {
> + virStorageReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> + _("storage pool does not support encrypted volumes"));
> + goto cleanup;
> + }
> +
> + if (rbd_create(ptr.ioctx, vol->name, vol->capacity,&order)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to create volume '%s/%s'"),
> + pool->def->source.name,
> + vol->name);
> + goto cleanup;
> + }
> +
> + if (volStorageBackendRBDRefreshVolInfo(vol, pool, ptr)< 0) {
> + goto cleanup;
> + }
> +
> + ret = 0;
> +
> +cleanup:
> + virStorageBackendRBDCloseRADOSConn(ptr);
> + return ret;
> +}
> +
> +static int virStorageBackendRBDRefreshVol(virConnectPtr conn,
> + virStoragePoolObjPtr pool ATTRIBUTE_UNUSED,
> + virStorageVolDefPtr vol)
> +{
> + virStorageBackendRBDStatePtr ptr;
> + ptr.cluster = NULL;
> + ptr.ioctx = NULL;
> + int ret = -1;
> +
> + if (virStorageBackendRBDOpenRADOSConn(&ptr, conn, pool)< 0) {
> + goto cleanup;
> + }
> +
> + if (rados_ioctx_create(ptr.cluster,
> + pool->def->source.name,&ptr.ioctx)< 0) {
> + virStorageReportError(VIR_ERR_INTERNAL_ERROR,
> + _("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
> + pool->def->source.name);
> + goto cleanup;
> + }
> +
> + if (volStorageBackendRBDRefreshVolInfo(vol, pool, ptr)< 0) {
> + goto cleanup;
> + }
> +
> + ret = 0;
> +
> +cleanup:
> + virStorageBackendRBDCloseRADOSConn(ptr);
> + return ret;
> +}
> +
> +virStorageBackend virStorageBackendRBD = {
> + .type = VIR_STORAGE_POOL_RBD,
> +
> + .refreshPool = virStorageBackendRBDRefreshPool,
> + .createVol = virStorageBackendRBDCreateVol,
> + .refreshVol = virStorageBackendRBDRefreshVol,
> + .deleteVol = virStorageBackendRBDDeleteVol,
> +};
> diff --git a/src/storage/storage_backend_rbd.h b/src/storage/storage_backend_rbd.h
> new file mode 100644
> index 0000000..2ae2513
> --- /dev/null
> +++ b/src/storage/storage_backend_rbd.h
> @@ -0,0 +1,30 @@
> +/*
> + * storage_backend_rbd.h: storage backend for RBD (RADOS Block Device) handling
> + *
> + * Copyright (C) 2012 Wido den Hollander
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library; if not, write to the Free Software
> + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
> + *
> + * Author: Wido den Hollander<wido at widodh.nl>
> + */
> +
> +#ifndef __VIR_STORAGE_BACKEND_RBD_H__
> +# define __VIR_STORAGE_BACKEND_RBD_H__
> +
> +# include "storage_backend.h"
> +
> +extern virStorageBackend virStorageBackendRBD;
> +
> +#endif /* __VIR_STORAGE_BACKEND_RBD_H__ */
> diff --git a/tests/storagepoolxml2xmlin/pool-rbd.xml b/tests/storagepoolxml2xmlin/pool-rbd.xml
> new file mode 100644
> index 0000000..c9d4790
> --- /dev/null
> +++ b/tests/storagepoolxml2xmlin/pool-rbd.xml
> @@ -0,0 +1,11 @@
> +<pool type='rbd'>
> +<name>ceph</name>
> +<source>
> +<name>rbd</name>
> +<host name='localhost' port='6789'/>
> +<host name='localhost' port='6790'/>
> +<auth username='admin' type='ceph'>
> + <secret uuid='2ec115d7-3a88-3ceb-bc12-0ac909a6fd87' usage='admin'/>
> +</auth>
> +</source>
> +</pool>
> diff --git a/tests/storagepoolxml2xmlout/pool-rbd.xml b/tests/storagepoolxml2xmlout/pool-rbd.xml
> new file mode 100644
> index 0000000..fa7fb34
> --- /dev/null
> +++ b/tests/storagepoolxml2xmlout/pool-rbd.xml
> @@ -0,0 +1,15 @@
> +<pool type='rbd'>
> +<name>ceph</name>
> +<uuid>47c1faee-0207-e741-f5ae-d9b019b98fe2</uuid>
> +<capacity unit='bytes'>0</capacity>
> +<allocation unit='bytes'>0</allocation>
> +<available unit='bytes'>0</available>
> +<source>
> +<host name='localhost' port='6789'/>
> +<host name='localhost' port='6790'/>
> +<name>rbd</name>
> +<auth username='admin' type='ceph'>
> +<secret uuid='2ec115d7-3a88-3ceb-bc12-0ac909a6fd87' usage='admin'/>
> +</auth>
> +</source>
> +</pool>
> diff --git a/tools/virsh.c b/tools/virsh.c
> index 8ee25c3..632c75e 100644
> --- a/tools/virsh.c
> +++ b/tools/virsh.c
> @@ -11894,6 +11894,10 @@ cmdVolInfo(vshControl *ctl, const vshCmd *cmd)
> vshPrint(ctl, "%-15s %s\n", _("Type:"), _("dir"));
> break;
>
> + case VIR_STORAGE_VOL_NETWORK:
> + vshPrint(ctl, "%-15s %s\n", _("Type:"), _("network"));
> + break;
> +
> default:
> vshPrint(ctl, "%-15s %s\n", _("Type:"), _("unknown"));
> }
> @@ -19852,6 +19856,9 @@ vshShowVersion(vshControl *ctl ATTRIBUTE_UNUSED)
> #ifdef WITH_STORAGE_LVM
> vshPrint(ctl, " LVM");
> #endif
> +#ifdef WITH_STORAGE_RBD
> + vshPrint(ctl, " RBD");
> +#endif
> vshPrint(ctl, "\n");
>
> vshPrint(ctl, "%s", _(" Miscellaneous:"));
More information about the libvir-list
mailing list