[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[libvirt] [PATCH RESEND RFC V2 2/2] Resctrl: Do cache allocation while boot a qemu VM



Set cachetune if user define cachetune in domain xml when using qemu
driver.

This patch adds 3 major private interface.

virResctrlGetFreeCache: return free cache, default cache substract cache
                        allocated.
virResctrlSetCachetunes: set cache banks which defined in a domain.
virResctrlRemoveCachetunes: remove cache allocation group from the
                            host.

Signed-off-by: Eli Qiao <liyong qiao intel com>
---
 include/libvirt/virterror.h               |   1 +
 src/Makefile.am                           |   1 +
 src/libvirt_private.syms                  |   9 +
 src/qemu/qemu_process.c                   |  81 +++
 src/util/virerror.c                       |   1 +
 src/util/virresctrl.c                     | 822 ++++++++++++++++++++++++++++++
 src/util/virresctrl.h                     |  88 ++++
 tests/Makefile.am                         |   8 +-
 tests/virresctrldata/L3-free.schemata     |   1 +
 tests/virresctrldata/L3CODE-free.schemata |   1 +
 tests/virresctrldata/L3DATA-free.schemata |   1 +
 tests/virresctrldata/linux-resctrl        |   1 +
 tests/virresctrldata/linux-resctrl-cdp    |   1 +
 tests/virresctrltest.c                    | 119 +++++
 14 files changed, 1134 insertions(+), 1 deletion(-)
 create mode 100644 src/util/virresctrl.c
 create mode 100644 src/util/virresctrl.h
 create mode 100644 tests/virresctrldata/L3-free.schemata
 create mode 100644 tests/virresctrldata/L3CODE-free.schemata
 create mode 100644 tests/virresctrldata/L3DATA-free.schemata
 create mode 120000 tests/virresctrldata/linux-resctrl
 create mode 120000 tests/virresctrldata/linux-resctrl-cdp
 create mode 100644 tests/virresctrltest.c

diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h
index 2efee8f..4bc0c74 100644
--- a/include/libvirt/virterror.h
+++ b/include/libvirt/virterror.h
@@ -132,6 +132,7 @@ typedef enum {
 
     VIR_FROM_PERF = 65,         /* Error from perf */
     VIR_FROM_LIBSSH = 66,       /* Error from libssh connection transport */
+    VIR_FROM_RESCTRL = 67,      /* Error from resctrl */
 
 # ifdef VIR_ENUM_SENTINELS
     VIR_ERR_DOMAIN_LAST
diff --git a/src/Makefile.am b/src/Makefile.am
index eae32dc..8dbb778 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -167,6 +167,7 @@ UTIL_SOURCES =							\
 		util/virprocess.c util/virprocess.h		\
 		util/virqemu.c util/virqemu.h			\
 		util/virrandom.h util/virrandom.c		\
+		util/virresctrl.h util/virresctrl.c		\
 		util/virrotatingfile.h util/virrotatingfile.c   \
 		util/virscsi.c util/virscsi.h			\
 		util/virscsihost.c util/virscsihost.h		\
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index c1e9471..72e5096 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -2439,6 +2439,15 @@ virRandomGenerateWWN;
 virRandomInt;
 
 
+# util/virresctrl.h
+virResctrlBitmap2String;
+virResctrlFreeSchemata;
+virResctrlGetFreeCache;
+virResctrlRemoveCachetunes;
+virResctrlSetCachetunes;
+virResctrlTypeToString;
+
+
 # util/virrotatingfile.h
 virRotatingFileReaderConsume;
 virRotatingFileReaderFree;
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index fa9990e..3b11caa 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -70,6 +70,7 @@
 #include "virbitmap.h"
 #include "viratomic.h"
 #include "virnuma.h"
+#include "virresctrl.h"
 #include "virstring.h"
 #include "virhostdev.h"
 #include "secret_util.h"
@@ -5130,6 +5131,77 @@ qemuProcessSetupVcpus(virDomainObjPtr vm)
     return 0;
 }
 
+static int
+qemuProcessSetCacheBanks(virCapsHostPtr caps, virDomainObjPtr vm)
+{
+    size_t i, j;
+    virDomainCachetunePtr cachetune;
+    virResctrlCachetunePtr resctrl_cachetune;
+    unsigned int max_vcpus = virDomainDefGetVcpusMax(vm->def);
+    pid_t *pids = NULL;
+    virDomainVcpuDefPtr vcpu;
+    size_t npids = 0;
+    size_t count = 0;
+    int ret = -1;
+
+    cachetune = &(vm->def->cachetune);
+
+    if (VIR_ALLOC_N(resctrl_cachetune, cachetune->n_banks) < 0)
+        goto cleanup;
+
+    /* construct resctrl_cachetune array */
+    for (i = 0; i < cachetune->n_banks; i++) {
+
+        resctrl_cachetune[i].cache_id = cachetune->cache_banks[i].cache_id;
+        resctrl_cachetune[i].type = cachetune->cache_banks[i].type;
+        resctrl_cachetune[i].size = cachetune->cache_banks[i].size;
+
+        /* get granularity from host's capabilities */
+        for (j = 0; j < caps->ncaches; j++) {
+            /* even enable CDP, granularity for code and data are same */
+            if (caps->caches[j]->id == resctrl_cachetune[i].cache_id &&
+                caps->caches[j]->controls) {
+                resctrl_cachetune[i].granularity = caps->caches[j]->controls[0]->granularity;
+                break;
+            }
+        }
+
+        /* create pids of vcpus array */
+        if (cachetune->cache_banks[i].vcpus) {
+            for (j = 0; j < max_vcpus; j++) {
+                if (virBitmapIsBitSet(cachetune->cache_banks[i].vcpus, j)) {
+
+                    vcpu = virDomainDefGetVcpu(vm->def, j);
+                    if (!vcpu->online)
+                        continue;
+
+                    if (VIR_RESIZE_N(pids, npids, count, 1) < 0)
+                        goto cleanup;
+                    pids[count ++] = qemuDomainGetVcpuPid(vm, j);
+                }
+            }
+        }
+    }
+
+    /* If not specify vcpus in cachetune, add vm->pid */
+    if (pids == NULL) {
+        if (VIR_ALLOC_N(pids, 1) < 0)
+            goto cleanup;
+        pids[0] = vm->pid;
+        count = 1;
+    }
+
+    ret = virResctrlSetCachetunes(vm->def->uuid,
+                                  resctrl_cachetune,
+                                  cachetune->n_banks,
+                                  pids,
+                                  count);
+
+ cleanup:
+    VIR_FREE(resctrl_cachetune);
+    VIR_FREE(pids);
+    return ret;
+}
 
 int
 qemuProcessSetupIOThread(virDomainObjPtr vm,
@@ -5966,6 +6038,12 @@ qemuProcessLaunch(virConnectPtr conn,
         qemuProcessAutoDestroyAdd(driver, vm, conn) < 0)
         goto cleanup;
 
+    VIR_DEBUG("Cache allocation");
+    if (vm->def->cachetune.n_banks > 0 &&
+        qemuProcessSetCacheBanks(&driver->caps->host,
+                                 vm) < 0)
+        goto cleanup;
+
     ret = 0;
 
  cleanup:
@@ -6472,6 +6550,9 @@ void qemuProcessStop(virQEMUDriverPtr driver,
     virPerfFree(priv->perf);
     priv->perf = NULL;
 
+    if (&(vm->def->cachetune) != NULL)
+        virResctrlRemoveCachetunes(vm->def->uuid);
+
     qemuProcessRemoveDomainStatus(driver, vm);
 
     /* Remove VNC and Spice ports from port reservation bitmap, but only if
diff --git a/src/util/virerror.c b/src/util/virerror.c
index ef17fb5..02fabcc 100644
--- a/src/util/virerror.c
+++ b/src/util/virerror.c
@@ -139,6 +139,7 @@ VIR_ENUM_IMPL(virErrorDomain, VIR_ERR_DOMAIN_LAST,
 
               "Perf", /* 65 */
               "Libssh transport layer",
+              "Resource Control",
     )
 
 
diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c
new file mode 100644
index 0000000..541c69c
--- /dev/null
+++ b/src/util/virresctrl.c
@@ -0,0 +1,822 @@
+/*
+ * virresctrl.c: methods for managing resource control
+ *
+ * Copyright (C) 2017 Intel, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ *  Eli Qiao <liyong qiao intel com>
+ */
+
+#include <config.h>
+#include <fcntl.h>
+#include <sys/file.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "viralloc.h"
+#include "virerror.h"
+#include "virfile.h"
+#include "virlog.h"
+#include "virresctrl.h"
+#include "virstring.h"
+#include "viruuid.h"
+
+VIR_LOG_INIT("util.resctrl");
+
+#define VIR_FROM_THIS VIR_FROM_RESCTRL
+#define SYSFS_RESCTRL_PATH "/sys/fs/resctrl"
+#define MAX_CBM_LEN 20
+#define VIR_RESCTRL_LOCK(fd, op) flock(fd, op)
+#define VIR_RESCTRL_UNLOCK(fd) flock(fd, LOCK_UN)
+#define CONSTRUCT_RESCTRL_PATH(domain_name, item_name) \
+do { \
+    if (NULL == domain_name) { \
+        if (virAsprintf(&path, "%s/%s", SYSFS_RESCTRL_PATH, item_name) < 0) \
+            return -1; \
+    } else { \
+        if (virAsprintf(&path, "%s/%s/%s", SYSFS_RESCTRL_PATH, domain_name, \
+                        item_name) < 0) \
+            return -1;  \
+    } \
+} while (0)
+
+VIR_ENUM_IMPL(virResctrl, VIR_RESCTRL_TYPE_LAST,
+              "L3",
+              "L3CODE",
+              "L3DATA")
+
+/**
+ * a virResctrlGroup represents a resource control group, it's a directory
+ * under /sys/fs/resctrl.
+ * e.g. /sys/fs/resctrl/CG1
+ * |-- cpus
+ * |-- schemata
+ * `-- tasks
+ * # cat schemata
+ * L3DATA:0=fffff;1=fffff
+ * L3CODE:0=fffff;1=fffff
+ *
+ * Besides, it can also represent the default resource control group of the
+ * host.
+ */
+
+typedef struct _virResctrlGroup virResctrlGroup;
+typedef virResctrlGroup *virResctrlGroupPtr;
+struct _virResctrlGroup {
+    char *name; /* resource group name, NULL for default host group */
+    size_t n_tasks; /* number of tasks assigned to the resource group */
+    char **tasks; /* task id list */
+    virResctrlSchemataPtr schemata[VIR_RESCTRL_TYPE_LAST]; /* Array for schemata */
+};
+
+/* All resource control groups on this host, including default resource group */
+typedef struct _virResctrlHost virResctrlHost;
+typedef virResctrlHost *virResctrlHostPtr;
+struct _virResctrlHost {
+    size_t n_groups; /* number of resource control group */
+    virResctrlGroupPtr *groups; /* list of resource control group */
+};
+
+void
+virResctrlFreeSchemata(virResctrlSchemataPtr ptr)
+{
+    size_t i;
+
+    if (!ptr)
+        return;
+
+    for (i = 0; i < ptr->n_masks; i++) {
+        virBitmapFree(ptr->masks[i]->mask);
+        VIR_FREE(ptr->masks[i]);
+    }
+
+    VIR_FREE(ptr);
+    ptr = NULL;
+}
+
+static void
+virResctrlFreeGroup(virResctrlGroupPtr ptr)
+{
+    size_t i;
+
+    if (!ptr)
+        return;
+
+    for (i = 0; i < ptr->n_tasks; i++)
+        VIR_FREE(ptr->tasks[i]);
+    VIR_FREE(ptr->name);
+
+    for (i = 0; i < VIR_RESCTRL_TYPE_LAST; i++)
+        virResctrlFreeSchemata(ptr->schemata[i]);
+
+    VIR_FREE(ptr);
+    ptr = NULL;
+}
+
+/* Return specify type of schemata string from schematalval.
+   e.g., 0=f;1=f */
+static int
+virResctrlGetSchemataString(virResctrlType type,
+                            const char *schemataval,
+                            char **schematastr)
+{
+    int rc = -1;
+    char *prefix = NULL;
+    char **lines = NULL;
+
+    if (virAsprintf(&prefix,
+                    "%s:",
+                    virResctrlTypeToString(type)) < 0)
+        return -1;
+
+    lines = virStringSplit(schemataval, "\n", 0);
+
+    if (VIR_STRDUP(*schematastr,
+                   virStringListGetFirstWithPrefix(lines, prefix)) < 0)
+        goto cleanup;
+
+    if (*schematastr == NULL)
+        rc = -1;
+    else
+        rc = 0;
+
+ cleanup:
+    VIR_FREE(prefix);
+    virStringListFree(lines);
+    return rc;
+}
+
+static int
+virResctrlRemoveSysGroup(const char* name)
+{
+    char *path = NULL;
+    int ret = -1;
+
+    if ((ret = virAsprintf(&path, "%s/%s", SYSFS_RESCTRL_PATH, name)) < 0)
+        return ret;
+
+    ret = rmdir(path);
+
+    VIR_FREE(path);
+    return ret;
+}
+
+static int
+virResctrlNewSysGroup(const char *name)
+{
+    char *path = NULL;
+    int ret = -1;
+    mode_t mode = 0755;
+
+    if (virAsprintf(&path, "%s/%s", SYSFS_RESCTRL_PATH, name) < 0)
+        return -1;
+
+    if (virDirCreate(path, mode, 0, 0, 0) < 0)
+        goto cleanup;
+
+    ret = 0;
+
+ cleanup:
+    VIR_FREE(path);
+    return ret;
+}
+
+static int
+virResctrlWrite(const char *name, const char *item, const char *content)
+{
+    char *path;
+    int writefd;
+    int rc = -1;
+
+    CONSTRUCT_RESCTRL_PATH(name, item);
+
+    if (!virFileExists(path))
+        goto cleanup;
+
+    if ((writefd = open(path, O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR)) < 0)
+        goto cleanup;
+
+    if (safewrite(writefd, content, strlen(content)) < 0)
+        goto cleanup;
+    rc = 0;
+
+ cleanup:
+    VIR_FREE(path);
+    VIR_FORCE_CLOSE(writefd);
+    return rc;
+}
+
+static
+virBitmapPtr virResctrlMask2Bitmap(const char *mask)
+{
+    virBitmapPtr bitmap;
+    unsigned int tmp;
+    size_t i;
+
+    if (virStrToLong_ui(mask, NULL, 16, &tmp) < 0)
+        return NULL;
+
+    bitmap = virBitmapNewEmpty();
+
+    for (i = 0; i < MAX_CBM_LEN; i++) {
+        if (((tmp & 0x1) == 0x1) &&
+                (virBitmapSetBitExpand(bitmap, i) < 0))
+            goto error;
+        tmp = tmp >> 1;
+    }
+    return bitmap;
+
+ error:
+    virBitmapFree(bitmap);
+    return NULL;
+}
+
+char *virResctrlBitmap2String(virBitmapPtr bitmap)
+{
+    char *tmp;
+    char *ret = NULL;
+    char *p;
+    tmp = virBitmapString(bitmap);
+    /* skip "0x" */
+    p = tmp + 2;
+
+    /* first non-0 position */
+    while (*++p == '0');
+
+    if (VIR_STRDUP(ret, p) < 0)
+        ret = NULL;
+
+    VIR_FREE(tmp);
+    return ret;
+}
+
+static int
+virResctrlParseSchemata(const char* schemata_str,
+                        virResctrlSchemataPtr schemata)
+{
+    VIR_DEBUG("schemata_str=%s, schemata=%p", schemata_str, schemata);
+
+    int ret = -1;
+    size_t i;
+    virResctrlMaskPtr mask;
+    char **schemata_list;
+    char *mask_str;
+
+    /* parse 0=fffff;1=f */
+    schemata_list = virStringSplit(schemata_str, ";", 0);
+
+    if (!schemata_list)
+        goto cleanup;
+
+    for (i = 0; schemata_list[i] != NULL; i++) {
+        /* parse 0=fffff */
+        mask_str = strchr(schemata_list[i], '=');
+
+        if (!mask_str)
+            goto cleanup;
+
+        if (VIR_ALLOC(mask) < 0)
+            goto cleanup;
+
+        mask->cache_id = i;
+        mask->mask = virResctrlMask2Bitmap(mask_str + 1);
+        schemata->n_masks += 1;
+        schemata->masks[i] = mask;
+
+    }
+    ret = 0;
+
+ cleanup:
+    virStringListFree(schemata_list);
+    return ret;
+}
+
+static int
+virResctrlLoadGroup(const char *name,
+                    virResctrlHostPtr host)
+{
+    VIR_DEBUG("name=%s, host=%p\n", name, host);
+
+    int ret = -1;
+    char *schemataval = NULL;
+    char *schemata_str = NULL;
+    virResctrlType i;
+    int rv;
+    virResctrlGroupPtr grp;
+    virResctrlSchemataPtr schemata;
+
+    rv = virFileReadValueString(&schemataval,
+                                SYSFS_RESCTRL_PATH "/%s/schemata",
+                                name ? name : "");
+
+    if (rv < 0)
+        return -1;
+
+    if (VIR_ALLOC(grp) < 0)
+        goto cleanup;
+
+    if (VIR_STRDUP(grp->name, name) < 0)
+        goto cleanup;
+
+    for (i = 0; i < VIR_RESCTRL_TYPE_LAST; i++) {
+        rv = virResctrlGetSchemataString(i, schemataval, &schemata_str);
+
+        if (rv < 0)
+            continue;
+
+        if (VIR_ALLOC(schemata) < 0)
+            goto cleanup;
+
+        schemata->type = i;
+
+        if (virResctrlParseSchemata(schemata_str, schemata) < 0) {
+            VIR_FREE(schemata);
+            VIR_FREE(schemata_str);
+            goto cleanup;
+        }
+
+        grp->schemata[i] = schemata;
+        VIR_FREE(schemata_str);
+    }
+
+    if (VIR_APPEND_ELEMENT(host->groups,
+                           host->n_groups,
+                           grp) < 0) {
+        virResctrlFreeGroup(grp);
+        goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    VIR_FREE(schemataval);
+    return ret;
+}
+
+static int
+virResctrlLoadHost(virResctrlHostPtr host)
+{
+    int rv = -1;
+    DIR *dirp = NULL;
+    char *path = NULL;
+    struct dirent *ent;
+
+    rv = virDirOpenIfExists(&dirp, SYSFS_RESCTRL_PATH);
+    if (rv < 0)
+        return -1;
+
+    /* load default group first */
+    if (virResctrlLoadGroup(NULL, host) < 0)
+        return -1;
+
+    while ((rv = virDirRead(dirp, &ent, path)) > 0) {
+        /* resctrl is not hierarchical, only read directory under
+           /sys/fs/resctrl */
+        if ((ent->d_type != DT_DIR) || STREQ(ent->d_name, "info"))
+            continue;
+
+        if (virResctrlLoadGroup(ent->d_name, host) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+static void
+virResctrlRefreshHost(virResctrlHostPtr host)
+{
+    virResctrlGroupPtr default_grp = NULL;
+    virResctrlSchemataPtr schemata = NULL;
+    size_t i, j;
+    virResctrlType t;
+
+    default_grp = host->groups[0];
+
+    for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+        if (default_grp->schemata[t] != NULL) {
+            for (i = 0; i < default_grp->schemata[t]->n_masks; i++) {
+                /* Reset default group's mask */
+                virBitmapSetAll(default_grp->schemata[t]->masks[i]->mask);
+                /* Loop each other resource group except default group */
+                for (j = 1; j < host->n_groups; j++) {
+                    schemata = host->groups[j]->schemata[t];
+                    virBitmapSubtract(default_grp->schemata[t]->masks[i]->mask,
+                                      schemata->masks[i]->mask);
+                }
+            }
+        }
+    }
+}
+
+static virResctrlGroupPtr
+virResctrlGetFreeGroup(void)
+{
+    size_t i;
+    virResctrlHostPtr host = NULL;
+    virResctrlGroupPtr grp = NULL;
+
+    if (VIR_ALLOC(host) < 0)
+        return NULL;
+
+    if (virResctrlLoadHost(host) < 0)
+        goto error;
+
+    virResctrlRefreshHost(host);
+
+    for (i = 1; i < host->n_groups; i++)
+        virResctrlFreeGroup(host->groups[i]);
+
+    grp = host->groups[0];
+    VIR_FREE(host);
+
+    return grp;
+
+ error:
+    virResctrlFreeGroup(grp);
+    return NULL;
+}
+
+virResctrlSchemataPtr
+virResctrlGetFreeCache(virResctrlType type)
+{
+    VIR_DEBUG("type=%d", type);
+
+    virResctrlType t;
+    virResctrlGroupPtr grp = NULL;
+    virResctrlSchemataPtr schemata = NULL;
+    int lockfd = -1;
+
+    lockfd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY);
+    if (lockfd < 0)
+        return NULL;
+
+    VIR_RESCTRL_LOCK(lockfd, LOCK_SH);
+
+    if ((grp = virResctrlGetFreeGroup()) == NULL)
+        goto cleanup;
+
+    for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+        if (t == type)
+            schemata = grp->schemata[t];
+        else
+            virResctrlFreeSchemata(grp->schemata[t]);
+    }
+
+ cleanup:
+    VIR_RESCTRL_UNLOCK(lockfd);
+    VIR_FORCE_CLOSE(lockfd);
+    return schemata;
+}
+
+static int
+virResctrlCalculateCbm(int cbm_len,
+                       virBitmapPtr defaultcbm,
+                       virBitmapPtr newcbm)
+{
+    VIR_DEBUG("cbm_len=%d, defaultcbm=%p, newcbm=%p",
+              cbm_len, defaultcbm, newcbm);
+
+    ssize_t pos = -1;
+    size_t i;
+
+    /* not enough cache way to be allocated */
+    if (virBitmapCountBits(defaultcbm) < cbm_len + 1)
+        return -1;
+
+    while ((pos = virBitmapNextSetBit(defaultcbm, pos)) >= 0) {
+        for (i = 0; i < cbm_len; i++)
+            ignore_value(virBitmapSetBitExpand(newcbm, i + pos));
+        /* Test if newcbm is sub set of defaultcbm */
+        if (virBitmapNextClearBit(defaultcbm, pos) > i + pos) {
+            break;
+        } else {
+            pos = pos + i - 1;
+            virBitmapClearAll(newcbm);
+        }
+    }
+
+    if (virBitmapCountBits(newcbm) != cbm_len)
+        return -1;
+
+    /* consume default cbm after allocation */
+    virBitmapSubtract(defaultcbm, newcbm);
+
+    return 0;
+}
+
+/* Fill mask value for newly created resource group base on hostcachebank
+ * and domcachebank */
+static int
+virResctrlFillMask(virResctrlGroupPtr grp,
+                  virResctrlGroupPtr free_grp,
+                  virResctrlCachetunePtr cachetune)
+{
+    VIR_DEBUG("grp=%p, free_grp=%p, cachetune=%p",
+              grp, free_grp, cachetune);
+
+    unsigned int cache_id;
+    unsigned int cache_type;
+    int cbm_candidate_len;
+    virResctrlSchemataPtr schemata;
+    virResctrlMaskPtr mask;
+
+    cache_type = cachetune->type;
+    cache_id = cachetune->cache_id;
+    schemata = grp->schemata[cache_type];
+
+    if ((schemata == NULL) && (VIR_ALLOC(schemata) < 0))
+        return -1;
+
+    if (VIR_ALLOC(mask) < 0)
+        return -1;
+
+    mask->cache_id = cache_id;
+    mask->mask = virBitmapNewEmpty();
+
+    /* here should be control->granularity and control->min
+       also domcachebank size should be checked while define domain xml */
+    cbm_candidate_len = cachetune->size / cachetune->granularity;
+    if (virResctrlCalculateCbm(cbm_candidate_len,
+                               free_grp->schemata[cache_type]->masks[cache_id]->mask,
+                               mask->mask) < 0)
+        goto error;
+
+    schemata->type = cache_type;
+    schemata->n_masks += 1;
+    schemata->masks[cache_id] = mask;
+    grp->schemata[cache_type] = schemata;
+
+    return 0;
+
+ error:
+    VIR_FREE(schemata);
+    return -1;
+}
+
+/* only keep the highest consecutive bits */
+static void
+virResctrlTrimMask(virResctrlMaskPtr mask)
+{
+    size_t i;
+    ssize_t setbit = -1;
+    ssize_t clearbit = -1;
+
+    clearbit = virBitmapNextClearBit(mask->mask, -1);
+    setbit = virBitmapNextSetBit(mask->mask, -1);
+    for (i = setbit; i < clearbit; i++)
+        ignore_value(virBitmapClearBit(mask->mask, i));
+}
+
+static int
+virResctrlCompleteMask(virResctrlSchemataPtr schemata,
+                       virResctrlSchemataPtr defaultschemata)
+{
+    size_t i;
+    virResctrlMaskPtr mask;
+
+    if (schemata == NULL && VIR_ALLOC(schemata) < 0)
+        return -1;
+
+    if (schemata->n_masks == defaultschemata->n_masks)
+        return 0;
+
+    for (i = 0; i < defaultschemata->n_masks; i++) {
+        if (schemata->masks[i] == NULL) {
+            if (VIR_ALLOC(mask) < 0)
+                goto error;
+
+            mask->cache_id = i;
+            mask->mask = virBitmapNewEmpty();
+            schemata->n_masks += 1;
+            schemata->masks[i] = mask;
+            /* resctrl doesn't allow mask to be zero
+               use higher bits to fill up the cbm which
+               domaincache bank doens't provide */
+            ignore_value(virBitmapSetBitExpand(mask->mask,
+                         virBitmapLastSetBit(defaultschemata->masks[i]->mask)));
+        }
+        /* only keep the highest consecutive bits for default group */
+        virResctrlTrimMask(defaultschemata->masks[i]);
+    }
+
+    return 0;
+
+ error:
+    VIR_FREE(schemata);
+    return -1;
+}
+
+/* complete the schemata in the resrouce group before it can be write back
+   to resctrl */
+static int
+virResctrlCompleteGroup(virResctrlGroupPtr grp,
+                        virResctrlGroupPtr default_grp)
+{
+    virResctrlType t;
+    virResctrlSchemataPtr schemata;
+    virResctrlSchemataPtr defaultschemata;
+
+    /* NOTES: resctrl system require we need provide all cache's cbm mask */
+    for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+        defaultschemata = default_grp->schemata[t];
+        if (defaultschemata != NULL) {
+            schemata = grp->schemata[t];
+            if (virResctrlCompleteMask(schemata, defaultschemata) < 0)
+                return -1;
+        }
+    }
+    return 0;
+}
+
+static
+char *virResctrlGetSchemataStr(virResctrlSchemataPtr schemata)
+{
+    virBuffer buf = VIR_BUFFER_INITIALIZER;
+    size_t i;
+
+    virBufferAsprintf(&buf, "%s:%u=%s",
+                      virResctrlTypeToString(schemata->type),
+                      schemata->masks[0]->cache_id,
+                      virResctrlBitmap2String(schemata->masks[0]->mask));
+
+    for (i = 1; i < schemata->n_masks; i ++)
+        virBufferAsprintf(&buf, ";%u=%s",
+                          schemata->masks[i]->cache_id,
+                          virResctrlBitmap2String(schemata->masks[i]->mask));
+
+    return virBufferContentAndReset(&buf);
+}
+
+static int
+virResctrlFlushGroup(virResctrlGroupPtr grp)
+{
+    int ret = -1;
+    size_t i;
+    char *schemata_str = NULL;
+    virResctrlType t;
+    virBuffer buf = VIR_BUFFER_INITIALIZER;
+
+    VIR_DEBUG("grp=%p", grp);
+
+    if (grp->name != NULL && virResctrlNewSysGroup(grp->name) < 0)
+        return -1;
+
+    for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+        if (grp->schemata[t] != NULL) {
+            schemata_str = virResctrlGetSchemataStr(grp->schemata[t]);
+            virBufferAsprintf(&buf, "%s\n", schemata_str);
+            VIR_FREE(schemata_str);
+        }
+    }
+
+    schemata_str = virBufferContentAndReset(&buf);
+
+    if (virResctrlWrite(grp->name, "schemata", schemata_str) < 0)
+        goto cleanup;
+
+    for (i = 0; i < grp->n_tasks; i++) {
+        if (virResctrlWrite(grp->name, "tasks", grp->tasks[i]) < 0)
+            goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    VIR_FREE(schemata_str);
+    return ret;
+}
+
+int virResctrlSetCachetunes(unsigned char* uuid,
+                            virResctrlCachetunePtr cachetunes, size_t ncachetune,
+                            pid_t *pids, size_t npid)
+{
+    size_t i;
+    int ret = -1;
+    char name[VIR_UUID_STRING_BUFLEN];
+    char *tmp;
+    int lockfd = -1;
+    virResctrlGroupPtr grp = NULL;
+    virResctrlGroupPtr default_grp = NULL;
+
+    virUUIDFormat(uuid, name);
+
+    if (ncachetune < 1)
+        return 0;
+
+    /* create new resource group */
+    if (VIR_ALLOC(grp) < 0)
+        goto error;
+
+    if (VIR_STRDUP(grp->name, name) < 0)
+        goto error;
+
+    /* allocate file lock */
+    lockfd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY);
+    if (lockfd < 0)
+        goto error;
+
+    VIR_RESCTRL_LOCK(lockfd, LOCK_EX);
+
+    if ((default_grp = virResctrlGetFreeGroup()) == NULL)
+        goto error;
+
+    /* Allocate cache for each cache bank defined in cache tune */
+    for (i = 0; i < ncachetune; i++) {
+        /* fill up newly crated grp and consume from default_grp */
+        if (virResctrlFillMask(grp, default_grp, &cachetunes[i]) < 0)
+            goto error;
+    }
+
+    /* Add tasks to grp */
+    for (i = 0; i < npid; i++) {
+        if (virAsprintf(&tmp, "%llu", (long long)pids[i]) < 0)
+            goto error;
+
+        if (VIR_APPEND_ELEMENT(grp->tasks,
+                               grp->n_tasks,
+                               tmp) < 0) {
+            VIR_FREE(tmp);
+            goto error;
+        }
+    }
+
+    if (virResctrlCompleteGroup(grp, default_grp) < 0) {
+        VIR_WARN("Failed to complete group");
+        goto error;
+    }
+
+    if (virResctrlFlushGroup(grp) < 0)
+        goto error;
+
+    if (virResctrlFlushGroup(default_grp) < 0) {
+        virResctrlRemoveSysGroup(grp->name);
+        goto error;
+    }
+
+    ret = 0;
+
+ error:
+    VIR_RESCTRL_UNLOCK(lockfd);
+    VIR_FORCE_CLOSE(lockfd);
+    virResctrlFreeGroup(grp);
+    virResctrlFreeGroup(default_grp);
+    return ret;
+}
+
+int virResctrlRemoveCachetunes(unsigned char* uuid)
+{
+    int ret = -1;
+    int lockfd = -1;
+    size_t i;
+    virResctrlType t;
+    virResctrlSchemataPtr schemata;
+    char name[VIR_UUID_STRING_BUFLEN];
+    virResctrlGroupPtr default_grp = NULL;
+
+    virUUIDFormat(uuid, name);
+
+    VIR_DEBUG("name=%s", name);
+
+    lockfd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY);
+    if (lockfd < 0)
+        return -1;
+
+    VIR_RESCTRL_LOCK(lockfd, LOCK_SH);
+
+    if (virResctrlRemoveSysGroup(name) < 0)
+        goto cleanup;
+
+    if ((default_grp = virResctrlGetFreeGroup()) == NULL)
+        goto cleanup;
+
+    for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+        schemata = default_grp->schemata[t];
+        if (schemata != NULL) {
+            for (i = 0; i < schemata->n_masks; i++)
+                virResctrlTrimMask(schemata->masks[i]);
+        }
+    }
+
+    if (virResctrlFlushGroup(default_grp) < 0)
+        goto cleanup;
+
+    ret = 0;
+
+ cleanup:
+    VIR_RESCTRL_UNLOCK(lockfd);
+    VIR_FORCE_CLOSE(lockfd);
+    return ret;
+}
diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h
new file mode 100644
index 0000000..49781ad
--- /dev/null
+++ b/src/util/virresctrl.h
@@ -0,0 +1,88 @@
+/*
+ * virresctrl.h: header for managing resctrl control
+ *
+ * Copyright (C) 2017 Intel, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ * Eli Qiao <liyong qiao intel com>
+ */
+
+#ifndef __VIR_RESCTRL_H__
+# define __VIR_RESCTRL_H__
+
+#include "virutil.h"
+#include "virbitmap.h"
+
+#define MAX_CACHE_ID 16
+
+typedef enum {
+    VIR_RESCTRL_TYPE_L3,
+    VIR_RESCTRL_TYPE_L3_CODE,
+    VIR_RESCTRL_TYPE_L3_DATA,
+
+    VIR_RESCTRL_TYPE_LAST
+} virResctrlType;
+
+VIR_ENUM_DECL(virResctrl);
+
+typedef struct _virResctrlCachetune virResctrlCachetune;
+typedef virResctrlCachetune *virResctrlCachetunePtr;
+struct _virResctrlCachetune {
+    unsigned int cache_id;
+    virResctrlType type;
+    unsigned long long size; /* in B */
+    unsigned long long granularity; /* min size of the cache */
+};
+
+/*
+ * a virResctrlMask represents one of mask object in a
+ * resource control group.
+ * e.g., 0=f
+ */
+typedef struct _virResctrlMask virResctrlMask;
+typedef virResctrlMask *virResctrlMaskPtr;
+struct _virResctrlMask {
+    unsigned int cache_id; /* cache resource id */
+    virBitmapPtr mask; /* the cbm mask */
+};
+
+/*
+ * a virResctrlSchemata represents schemata objects of specific type of
+ * resource in a resource control group.
+ * eg: L3:0=f,1=ff
+ */
+typedef struct _virResctrlSchemata virResctrlSchemata;
+typedef virResctrlSchemata *virResctrlSchemataPtr;
+struct _virResctrlSchemata {
+    virResctrlType type; /* resource control type, e.g., L3 */
+    size_t n_masks; /* number of masks */
+    virResctrlMaskPtr masks[MAX_CACHE_ID]; /* array of mask, use array for easy index */
+};
+
+/* Get free cache of the host, result saved in schemata */
+virResctrlSchemataPtr virResctrlGetFreeCache(virResctrlType type);
+
+/* Get mask string from Bitmap */
+char *virResctrlBitmap2String(virBitmapPtr bitmap);
+
+void virResctrlFreeSchemata(virResctrlSchemataPtr ptr);
+
+int virResctrlSetCachetunes(unsigned char* uuid,
+                            virResctrlCachetunePtr cachetunes, size_t ncachetune,
+                            pid_t *pids, size_t npid);
+int virResctrlRemoveCachetunes(unsigned char* uuid);
+#endif
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 19986dc..e0b9923 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -229,6 +229,7 @@ if WITH_LINUX
 test_programs += fchosttest
 test_programs += scsihosttest
 test_programs += vircaps2xmltest
+test_programs += virresctrltest
 test_libraries += virusbmock.la \
 	virnetdevbandwidthmock.la \
 	virnumamock.la \
@@ -1149,8 +1150,13 @@ virnumamock_la_CFLAGS = $(AM_CFLAGS)
 virnumamock_la_LDFLAGS = $(MOCKLIBS_LDFLAGS)
 virnumamock_la_LIBADD = $(MOCKLIBS_LIBS)
 
+virresctrltest_SOURCES = \
+	virresctrltest.c testutils.h testutils.c virfilewrapper.h virfilewrapper.c
+virresctrltest_LDADD = $(LDADDS)
+
 else ! WITH_LINUX
-EXTRA_DIST += vircaps2xmltest.c virnumamock.c virfilewrapper.c virfilewrapper.h
+EXTRA_DIST += vircaps2xmltest.c virnumamock.c virfilewrapper.c \
+			  virfilewrapper.h virresctrltest.c
 endif ! WITH_LINUX
 
 if WITH_NSS
diff --git a/tests/virresctrldata/L3-free.schemata b/tests/virresctrldata/L3-free.schemata
new file mode 100644
index 0000000..9b47d25
--- /dev/null
+++ b/tests/virresctrldata/L3-free.schemata
@@ -0,0 +1 @@
+L3:0=1ffff;1=1ffff
diff --git a/tests/virresctrldata/L3CODE-free.schemata b/tests/virresctrldata/L3CODE-free.schemata
new file mode 100644
index 0000000..7039c45
--- /dev/null
+++ b/tests/virresctrldata/L3CODE-free.schemata
@@ -0,0 +1 @@
+L3CODE:0=cffff;1=cffff
diff --git a/tests/virresctrldata/L3DATA-free.schemata b/tests/virresctrldata/L3DATA-free.schemata
new file mode 100644
index 0000000..30f1cbd
--- /dev/null
+++ b/tests/virresctrldata/L3DATA-free.schemata
@@ -0,0 +1 @@
+L3DATA:0=3ffff;1=3ffff
diff --git a/tests/virresctrldata/linux-resctrl b/tests/virresctrldata/linux-resctrl
new file mode 120000
index 0000000..069dfb2
--- /dev/null
+++ b/tests/virresctrldata/linux-resctrl
@@ -0,0 +1 @@
+../vircaps2xmldata/linux-resctrl
\ No newline at end of file
diff --git a/tests/virresctrldata/linux-resctrl-cdp b/tests/virresctrldata/linux-resctrl-cdp
new file mode 120000
index 0000000..c5a973f
--- /dev/null
+++ b/tests/virresctrldata/linux-resctrl-cdp
@@ -0,0 +1 @@
+../vircaps2xmldata/linux-resctrl-cdp
\ No newline at end of file
diff --git a/tests/virresctrltest.c b/tests/virresctrltest.c
new file mode 100644
index 0000000..a9d75f1
--- /dev/null
+++ b/tests/virresctrltest.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) Intel, Inc. 2017
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ *      Eli Qiao <liyong qiao intel com>
+ */
+
+#include <config.h>
+#include <stdlib.h>
+
+#include "testutils.h"
+#include "virbitmap.h"
+#include "virfilewrapper.h"
+#include "virresctrl.h"
+
+
+#define VIR_FROM_THIS VIR_FROM_NONE
+
+struct virResctrlData {
+    const char *filename;
+    virResctrlType type;
+};
+
+static void
+GetSchemataStr(virResctrlSchemataPtr schemata, char **str)
+{
+    size_t i;
+
+    virBuffer buf = VIR_BUFFER_INITIALIZER;
+    virBufferAsprintf(&buf, "%s:%u=%s",
+                            virResctrlTypeToString(schemata->type),
+                            schemata->masks[0]->cache_id,
+                            virResctrlBitmap2String(schemata->masks[0]->mask));
+
+    for (i = 1; i < schemata->n_masks; i ++)
+        virBufferAsprintf(&buf, ";%u=%s",
+                                schemata->masks[i]->cache_id,
+                                virResctrlBitmap2String(schemata->masks[i]->mask));
+
+    *str = virBufferContentAndReset(&buf);
+}
+
+static int
+test_virResctrl(const void *opaque)
+{
+    struct virResctrlData *data = (struct virResctrlData *) opaque;
+    char *dir = NULL;
+    char *resctrl = NULL;
+    int ret = -1;
+    virResctrlSchemataPtr schemata = NULL;
+    char *schemata_str = NULL;
+    char *schemata_file;
+
+    if (virAsprintf(&resctrl, "%s/virresctrldata/linux-%s/resctrl",
+                    abs_srcdir, data->filename) < 0)
+        goto cleanup;
+
+    if (virAsprintf(&schemata_file, "%s/virresctrldata/%s-free.schemata",
+                    abs_srcdir, virResctrlTypeToString(data->type)) < 0)
+        goto cleanup;
+
+    if (virFileWrapperAddPrefix("/sys/fs/resctrl", resctrl) < 0)
+        goto cleanup;
+
+    if ((schemata = virResctrlGetFreeCache(data->type)) == NULL)
+        goto cleanup;
+
+    virFileWrapperClearPrefixes();
+
+    GetSchemataStr(schemata, &schemata_str);
+
+    if (virTestCompareToFile(schemata_str, schemata_file) < 0)
+        goto cleanup;
+
+    ret = 0;
+
+ cleanup:
+    VIR_FREE(dir);
+    VIR_FREE(resctrl);
+    VIR_FREE(schemata_str);
+    virResctrlFreeSchemata(schemata);
+    return ret;
+}
+
+static int
+mymain(void)
+{
+    int ret = 0;
+
+#define DO_TEST_FULL(filename, type) \
+    do {                                                                \
+        struct virResctrlData data = {filename,                         \
+                                      type};                            \
+        if (virTestRun(filename, test_virResctrl, &data) < 0)           \
+            ret = -1;                                                   \
+    } while (0)
+
+    DO_TEST_FULL("resctrl", VIR_RESCTRL_TYPE_L3);
+    DO_TEST_FULL("resctrl-cdp", VIR_RESCTRL_TYPE_L3_CODE);
+    DO_TEST_FULL("resctrl-cdp", VIR_RESCTRL_TYPE_L3_DATA);
+
+    return ret;
+}
+
+VIR_TEST_MAIN(mymain)
-- 
1.9.1


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]