[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [Libvir] PATCH: 13/16: iSCSI backend



This provides a storage pool using the iSCSI protocol. Since there
is no API for iSCSI it is implemented by simply shelling out to the
iscsiadm command line tool. A pool corresponds to a single target
on the iSCSI server. Starting a pool logs into the server and maps
the target's LUNs into the local filesystem. The default nodes are
under /dev, allocated-on-demand and thus not guarenteed to be stable
across reboots. For this reason it is recommended by the pool target
path be configured to point to /dev/disk/by-path or /dev/disk/by-id
whose entries are guarenteed stable for lifetime of the target+LUN.
The 'refresh' operation will rescan the target for new LUNs and purge
old LUNs allowing dynamic updates without needing a pool restart.

 b/src/storage_backend_iscsi.c |  424 ++++++++++++++++++++++++++++++++++++++++++
 b/src/storage_backend_iscsi.h |   45 ++++
 configure.in                  |   22 ++
 libvirt.spec.in               |    4 
 src/Makefile.am               |    6 
 src/storage_backend.c         |   15 +
 6 files changed, 516 insertions(+)

diff -r 9a3200af0a3d configure.in
--- a/configure.in	Thu Feb 07 11:14:56 2008 -0500
+++ b/configure.in	Thu Feb 07 11:34:46 2008 -0500
@@ -561,6 +561,8 @@ AC_ARG_WITH(storage-fs,
 [  --with-storage-fs           with FileSystem backend for the storage driver (on)],[],[with_storage_fs=check])
 AC_ARG_WITH(storage-lvm,
 [  --with-storage-lvm          with LVM backend for the storage driver (on)],[],[with_storage_lvm=check])
+AC_ARG_WITH(storage-iscsi,
+[  --with-storage-iscsi        with iSCSI backend for the storage driver (on)],[],[with_storage_iscsi=check])
 
 if test "$with_storage_fs" = "yes" -o "$with_storage_fs" = "check"; then
   AC_PATH_PROG(MOUNT, [mount], [], [$PATH:/sbin:/usr/sbin])
@@ -653,6 +655,25 @@ if test "$with_storage_lvm" = "yes" -o "
   fi
 fi
 AM_CONDITIONAL(WITH_STORAGE_LVM, [test "$with_storage_lvm" = "yes"])
+
+
+
+if test "$with_storage_iscsi" = "yes" -o "$with_storage_iscsi" = "check"; then
+  AC_PATH_PROG(ISCSIADM, [iscsiadm], [], [$PATH:/sbin:/usr/sbin])
+  if test "$with_storage_iscsi" = "yes" ; then
+    if test -z "$ISCSIADM" ; then AC_MSG_ERROR(We need iscsiadm for iSCSI storage driver) ; fi
+  else
+    if test -z "$ISCSIADM" ; then with_storage_iscsi=no ; fi
+
+    if test "$with_storage_iscsi" = "check" ; then with_storage_iscsi=yes ; fi
+  fi
+
+  if test "$with_storage_iscsi" = "yes" ; then
+    AC_DEFINE_UNQUOTED(WITH_STORAGE_ISCSI, 1, [whether iSCSI backend for storage driver is enabled])
+    AC_DEFINE_UNQUOTED([ISCSIADM],["$ISCSIADM"],[Location of iscsiadm program])
+  fi
+fi
+AM_CONDITIONAL(WITH_STORAGE_ISCSI, [test "$with_storage_iscsi" = "yes"])
 
 
 dnl
@@ -869,6 +890,7 @@ AC_MSG_NOTICE([      FS: $with_storage_f
 AC_MSG_NOTICE([      FS: $with_storage_fs])
 AC_MSG_NOTICE([   NetFS: $with_storage_fs])
 AC_MSG_NOTICE([     LVM: $with_storage_lvm])
+AC_MSG_NOTICE([   iSCSI: $with_storage_iscsi])
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([Libraries])
 AC_MSG_NOTICE([])
diff -r 9a3200af0a3d libvirt.spec.in
--- a/libvirt.spec.in	Thu Feb 07 11:14:56 2008 -0500
+++ b/libvirt.spec.in	Thu Feb 07 11:34:46 2008 -0500
@@ -51,6 +51,8 @@ Requires: /usr/sbin/qcow-create
 %endif
 # For LVM drivers
 Requires: lvm2
+# For ISCSI driver
+Requires: iscsi-initiator-utils
 BuildRequires: xen-devel
 BuildRequires: libxml2-devel
 BuildRequires: readline-devel
@@ -77,6 +79,8 @@ BuildRequires: /usr/sbin/qcow-create
 %endif
 # For LVM drivers
 BuildRequires: lvm2
+# For ISCSI driver
+BuildRequires: iscsi-initiator-utils
 Obsoletes: libvir
 ExclusiveArch: i386 x86_64 ia64
 
diff -r 9a3200af0a3d src/Makefile.am
--- a/src/Makefile.am	Thu Feb 07 11:14:56 2008 -0500
+++ b/src/Makefile.am	Thu Feb 07 11:34:46 2008 -0500
@@ -75,6 +75,12 @@ EXTRA_DIST += storage_backend_logical.h 
 EXTRA_DIST += storage_backend_logical.h storage_backend_logical.c
 endif
 
+if WITH_STORAGE_ISCSI
+CLIENT_SOURCES +=  storage_backend_iscsi.h storage_backend_iscsi.c
+else
+EXTRA_DIST +=  storage_backend_iscsi.h storage_backend_iscsi.c
+endif
+
 
 
 libvirt_la_SOURCES = $(CLIENT_SOURCES) $(SERVER_SOURCES)
diff -r 9a3200af0a3d src/storage_backend.c
--- a/src/storage_backend.c	Thu Feb 07 11:14:56 2008 -0500
+++ b/src/storage_backend.c	Thu Feb 07 11:34:46 2008 -0500
@@ -39,6 +39,10 @@
 #if WITH_STORAGE_LVM
 #include "storage_backend_logical.h"
 #endif
+#if WITH_STORAGE_ISCSI
+#include "storage_backend_iscsi.h"
+#endif
+
 
 #include "util.h"
 
@@ -53,6 +57,9 @@ static virStorageBackendPtr backends[] =
 #endif
 #if WITH_STORAGE_LVM
     &virStorageBackendLogical,
+#endif
+#if WITH_STORAGE_ISCSI
+    &virStorageBackendISCSI,
 #endif
 };
 
@@ -94,6 +101,10 @@ int virStorageBackendFromString(const ch
 #if WITH_STORAGE_LVM
     if (STREQ(type, "logical"))
         return VIR_STORAGE_POOL_LOGICAL;
+#endif
+#if WITH_STORAGE_ISCSI
+    if (STREQ(type, "iscsi"))
+        return VIR_STORAGE_POOL_ISCSI;
 #endif
     virStorageReportError(NULL, VIR_ERR_INTERNAL_ERROR, "unknown storage backend type %s", type);
     return -1;
@@ -112,6 +123,10 @@ const char *virStorageBackendToString(in
 #if WITH_STORAGE_LVM
     case VIR_STORAGE_POOL_LOGICAL:
         return "logical";
+#endif
+#if WITH_STORAGE_ISCSI
+    case VIR_STORAGE_POOL_ISCSI:
+        return "iscsi";
 #endif
     }
 
diff -r 9a3200af0a3d src/storage_backend_iscsi.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/storage_backend_iscsi.c	Thu Feb 07 11:34:46 2008 -0500
@@ -0,0 +1,424 @@
+/*
+ * storage_backend_iscsi.c: storage backend for iSCSI handling
+ *
+ * Copyright (C) 2007-2008 Red Hat, Inc.
+ * Copyright (C) 2007-2008 Daniel P. Berrange
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+ *
+ * Author: Daniel P. Berrange <berrange redhat com>
+ */
+
+#include <config.h>
+
+#include <sys/socket.h>
+#include <netdb.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <string.h>
+#include <stdio.h>
+#include <regex.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <dirent.h>
+
+#include "storage_backend_iscsi.h"
+#include "util.h"
+
+static int virStorageBackendISCSITargetIP(virConnectPtr conn,
+                                          const char *hostname,
+                                          char *ipaddr,
+                                          size_t ipaddrlen)
+{
+    struct addrinfo hints;
+    struct addrinfo *result = NULL;
+    int ret;
+
+    memset(&hints, 0, sizeof hints);
+    hints.ai_flags = AI_ADDRCONFIG;
+    hints.ai_family = AF_INET;
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_protocol = 0;
+
+    ret = getaddrinfo(hostname, NULL, &hints, &result);
+    if (ret != 0) {
+        virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, "getaddrinfo %s",
+                              gai_strerror(ret));
+        return -1;
+    }
+
+    if (result == NULL) {
+        virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR,
+                              "no IP address for target %s", hostname);
+        return -1;
+    }
+
+    if (getnameinfo(result->ai_addr, result->ai_addrlen,
+                    ipaddr, ipaddrlen, NULL, 0,
+                    NI_NUMERICHOST) < 0) {
+        virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR,
+                              "cannot format ip addr for %s", hostname);
+        freeaddrinfo(result);
+        return -1;
+    }
+
+    freeaddrinfo(result);
+    return 0;
+}
+
+static int virStorageBackendISCSIExtractSession(virConnectPtr conn,
+                                                virStoragePoolObjPtr pool,
+                                                char **const groups,
+                                                void *data)
+{
+    char **session = data;
+
+    if (STREQ(groups[1], pool->def->source.devices[0].path)) {
+        if ((*session = strdup(groups[0])) == NULL) {
+            virStorageReportError(conn, VIR_ERR_NO_MEMORY, "session");
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+static char *virStorageBackendISCSISession(virConnectPtr conn,
+                                           virStoragePoolObjPtr pool)
+{
+    /*
+     * # iscsiadm --mode session  -P 0
+     * tcp: [1] 192.168.122.170:3260,1 demo-tgt-b
+     * tcp: [2] 192.168.122.170:3260,1 demo-tgt-a
+     *
+     * Pull out 2nd and 4th fields
+     */
+    const char *regexes[] = {
+        "^tcp:\\s+\\[(\\S+)\\]\\s+\\S+\\s+(\\S+)\\s*$"
+    };
+    int vars[] = {
+        2,
+    };
+    const char *prog[] = {
+        ISCSIADM, "--mode", "session", "-P", "0", NULL
+    };
+    char *session = NULL;
+
+    if (virStorageBackendRunProgRegex(conn, pool,
+                                      prog,
+                                      1,
+                                      regexes,
+                                      vars,
+                                      virStorageBackendISCSIExtractSession,
+                                      &session) < 0)
+        return NULL;
+
+    if (session == NULL) {
+        virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, "cannot find session");
+        return NULL;
+    }
+
+    return session;
+}
+
+static int virStorageBackendISCSIConnection(virConnectPtr conn,
+                                            virStoragePoolObjPtr pool,
+                                            const char *portal,
+                                            const char *action)
+{
+    const char *cmdargv[] = {
+        ISCSIADM, "--mode", "node", "--portal", portal,
+        "--targetname", pool->def->source.devices[0].path, action, NULL
+    };
+
+    if (virRun(conn, (char **)cmdargv, NULL) < 0)
+        return -1;
+
+    return 0;
+}
+
+
+static int virStorageBackendISCSIMakeLUN(virConnectPtr conn,
+                                         virStoragePoolObjPtr pool,
+                                         char **const groups,
+                                         void *data ATTRIBUTE_UNUSED)
+{
+    virStorageVolDefPtr vol;
+    int fd = -1;
+    char scsiid[100];
+    char *dev = groups[4];
+    int opentries = 0;
+    char *devpath = NULL;
+
+    snprintf(scsiid, sizeof(scsiid)-1, "%s:%s:%s:%s",
+             groups[0], groups[1], groups[2], groups[3]);
+
+    if ((vol = calloc(1, sizeof(virStorageVolDef))) == NULL) {
+        virStorageReportError(conn, VIR_ERR_NO_MEMORY, "volume");
+        return -1;
+    }
+
+    if ((vol->name = strdup(scsiid)) == NULL) {
+        virStorageReportError(conn, VIR_ERR_NO_MEMORY, "name");
+        goto cleanup;
+    }
+
+    if ((devpath = malloc(5 + strlen(dev) + 1)) == NULL) {
+        virStorageReportError(conn, VIR_ERR_NO_MEMORY, "devpath");
+        goto cleanup;
+    }
+    strcpy(devpath, "/dev/");
+    strcat(devpath, dev);
+    /* It can take a little while between logging into the ISCSI
+     * server and udev creating the /dev nodes, so if we get ENOENT
+     * we must retry a few times - they should eventually appear.
+     * We currently wait for upto 5 seconds. Is this good enough ?
+     * Perhaps not on a very heavily loaded system Any other
+     * options... ?
+     */
+ reopen:
+    if ((fd = open(devpath, O_RDONLY)) < 0) {
+        opentries++;
+        if (errno == ENOENT && opentries < 50) {
+            usleep(100 * 1000);
+            goto reopen;
+        }
+        virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR,
+                              "cannot open %s: %s (%d)",
+                              devpath, strerror(errno), errno);
+        goto cleanup;
+    }
+
+    /* Now figure out the stable path
+     *
+     * XXX this method is O(N) because it scans the pool target
+     * dir every time its run. Should figure out a more efficient
+     * way of doing this...
+     */
+    if ((vol->target.path = virStorageBackendStablePath(conn, pool, devpath)) == NULL)
+        goto cleanup;
+
+    if (devpath != vol->target.path)
+        free(devpath);
+    devpath = NULL;
+
+    if (virStorageBackendUpdateVolInfoFD(conn, vol, fd, 1) < 0)
+        goto cleanup;
+
+    /* XXX use unique iSCSI id instead */
+    vol->key = strdup(vol->target.path);
+    if (vol->key == NULL) {
+        virStorageReportError(conn, VIR_ERR_NO_MEMORY, "key");
+        goto cleanup;
+    }
+
+
+    pool->def->capacity += vol->capacity;
+    pool->def->allocation += vol->allocation;
+
+    vol->next = pool->volumes;
+    pool->volumes = vol;
+    pool->nvolumes++;
+
+    close(fd);
+
+    return 0;
+
+ cleanup:
+    if (fd != -1) close(fd);
+    free(devpath);
+    virStorageVolDefFree(vol);
+    return -1;
+}
+
+static int virStorageBackendISCSIFindLUNs(virConnectPtr conn,
+                                          virStoragePoolObjPtr pool,
+                                          const char *session)
+{
+    /*
+     * # iscsiadm --mode session -r $session -P 3
+     *
+     *           scsi1 Channel 00 Id 0 Lun: 0
+     *           scsi1 Channel 00 Id 0 Lun: 1
+     *                   Attached scsi disk sdc          State: running
+     *           scsi1 Channel 00 Id 0 Lun: 2
+     *                   Attached scsi disk sdd          State: running
+     *           scsi1 Channel 00 Id 0 Lun: 3
+     *                   Attached scsi disk sde          State: running
+     *           scsi1 Channel 00 Id 0 Lun: 4
+     *                   Attached scsi disk sdf          State: running
+     *           scsi1 Channel 00 Id 0 Lun: 5
+     *                   Attached scsi disk sdg          State: running
+     *
+     * Need 2 regex to match alternating lines
+     */
+    const char *regexes[] = {
+        "^\\s*scsi(\\S+)\\s+Channel\\s+(\\S+)\\s+Id\\s+(\\S+)\\s+Lun:\\s+(\\S+)\\s*$",
+        "^\\s*Attached\\s+scsi\\s+disk\\s+(\\S+)\\s+State:\\s+running\\s*$"
+    };
+    int vars[] = {
+        4, 1
+    };
+    const char *prog[] = {
+        ISCSIADM, "--mode", "session", "-r", session, "-P", "3", NULL,
+    };
+
+    return virStorageBackendRunProgRegex(conn, pool,
+                                         prog,
+                                         2,
+                                         regexes,
+                                         vars,
+                                         virStorageBackendISCSIMakeLUN,
+                                         NULL);
+}
+
+
+static int virStorageBackendISCSILogin(virConnectPtr conn,
+                                       virStoragePoolObjPtr pool,
+                                       const char *portal)
+{
+    return virStorageBackendISCSIConnection(conn, pool, portal, "--login");
+}
+
+static int virStorageBackendISCSILogout(virConnectPtr conn,
+                                        virStoragePoolObjPtr pool,
+                                        const char *portal)
+{
+    return virStorageBackendISCSIConnection(conn, pool, portal, "--logout");
+}
+
+static char *virStorageBackendISCSIPortal(virConnectPtr conn,
+                                          virStoragePoolObjPtr pool)
+{
+    char ipaddr[NI_MAXHOST];
+    char *portal;
+
+    if (virStorageBackendISCSITargetIP(conn,
+                                       pool->def->source.host.name,
+                                       ipaddr, sizeof(ipaddr)) < 0)
+        return NULL;
+
+    portal = malloc(strlen(ipaddr) + 1 + 4 + 2 + 1);
+    if (portal == NULL) {
+        virStorageReportError(conn, VIR_ERR_NO_MEMORY, "portal");
+        return NULL;
+    }
+
+    strcpy(portal, ipaddr);
+    strcat(portal, ":3260,1");
+
+    return portal;
+}
+
+
+static int virStorageBackendISCSIStartPool(virConnectPtr conn,
+                                           virStoragePoolObjPtr pool)
+{
+    char *portal = NULL;
+
+    if (pool->def->source.host.name == NULL) {
+        virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, "missing source host");
+        return -1;
+    }
+
+    if (pool->def->source.ndevice != 1 ||
+        pool->def->source.devices[0].path == NULL) {
+        virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, "missing source device");
+        return -1;
+    }
+
+    if ((portal = virStorageBackendISCSIPortal(conn, pool)) == NULL)
+        return -1;
+    if (virStorageBackendISCSILogin(conn, pool, portal) < 0) {
+        free(portal);
+        return -1;
+    }
+    free(portal);
+    return 0;
+}
+
+static int virStorageBackendISCSIRefreshPool(virConnectPtr conn,
+                                             virStoragePoolObjPtr pool)
+{
+    char *portal = NULL, *session = NULL;
+
+    pool->def->allocation = pool->def->capacity = pool->def->available = 0;
+
+    if ((portal = virStorageBackendISCSIPortal(conn, pool)) == NULL)
+        return -1;
+
+    if ((session = virStorageBackendISCSISession(conn, pool)) == NULL) {
+        goto cleanup;
+    }
+    if (virStorageBackendISCSIFindLUNs(conn, pool, session) < 0)
+        goto cleanup;
+    free(portal);
+    free(session);
+
+    return 0;
+
+ cleanup:
+    free(portal);
+    free(session);
+    return -1;
+}
+
+
+static int virStorageBackendISCSIStopPool(virConnectPtr conn,
+                                          virStoragePoolObjPtr pool)
+{
+    char *portal;
+
+    if ((portal = virStorageBackendISCSIPortal(conn, pool)) == NULL)
+        return -1;
+
+    if (virStorageBackendISCSILogout(conn, pool, portal) < 0) {
+        free(portal);
+        return -1;
+    }
+    free(portal);
+
+    return 0;
+}
+
+
+virStorageBackend virStorageBackendISCSI = {
+  .type = VIR_STORAGE_POOL_ISCSI,
+
+  .startPool = virStorageBackendISCSIStartPool,
+  .refreshPool = virStorageBackendISCSIRefreshPool,
+  .stopPool = virStorageBackendISCSIStopPool,
+
+  .poolOptions = {
+        .flags = (VIR_STORAGE_BACKEND_POOL_SOURCE_HOST |
+                  VIR_STORAGE_BACKEND_POOL_SOURCE_DEVICE)
+    },
+
+  .volType = VIR_STORAGE_VOL_BLOCK,
+};
+
+/*
+ * vim: set tabstop=4:
+ * vim: set shiftwidth=4:
+ * vim: set expandtab:
+ */
+/*
+ * Local variables:
+ *  indent-tabs-mode: nil
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ *  tab-width: 4
+ * End:
+ */
diff -r 9a3200af0a3d src/storage_backend_iscsi.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/storage_backend_iscsi.h	Thu Feb 07 11:34:46 2008 -0500
@@ -0,0 +1,45 @@
+/*
+ * storage_backend_iscsi.h: storage backend for iSCSI handling
+ *
+ * Copyright (C) 2007-2008 Red Hat, Inc.
+ * Copyright (C) 2007-2008 Daniel P. Berrange
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+ *
+ * Author: Daniel P. Berrange <berrange redhat com>
+ */
+
+#ifndef __VIR_STORAGE_BACKEND_ISCSI_H__
+#define __VIR_STORAGE_BACKEND_ISCSI_H__
+
+#include "storage_backend.h"
+
+extern virStorageBackend virStorageBackendISCSI;
+
+#endif /* __VIR_STORAGE_BACKEND_ISCSI_H__ */
+
+/*
+ * vim: set tabstop=4:
+ * vim: set shiftwidth=4:
+ * vim: set expandtab:
+ */
+/*
+ * Local variables:
+ *  indent-tabs-mode: nil
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ *  tab-width: 4
+ * End:
+ */

-- 
|=- Red Hat, Engineering, Emerging Technologies, Boston.  +1 978 392 2496 -=|
|=-           Perl modules: http://search.cpan.org/~danberr/              -=|
|=-               Projects: http://freshmeat.net/~danielpb/               -=|
|=-  GnuPG: 7D3B9505   F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505  -=| 


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]