[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [libvirt] [PATCH 04/13] Enable cpuset cgroup and synchronous vcpupin info to cgroup.



On Tue, Jul 10, 2012 at 05:14:42PM +0800, tangchen wrote:
> vcpu threads pin are implemented using sched_setaffinity(), but not controlled by cgroup.
> This patch does the following things:
>     1) enable cpuset cgroup

I think it's better to make this part and adding of
virCgroupGetCpusetCpus/virCgroupSetCpusetCpus as a seperate
patch.

>     2) reflect all the vcpu threads pin info to cgroup
> 
> Signed-off-by: Tang Chen <tangchen cn fujitsu com>
> ---
>  src/libvirt_private.syms |    2 ++
>  src/qemu/qemu_cgroup.c   |   44 ++++++++++++++++++++++++++++++++++++++++++++
>  src/qemu/qemu_cgroup.h   |    2 ++
>  src/qemu/qemu_driver.c   |   38 ++++++++++++++++++++++++++++++--------
>  src/util/cgroup.c        |   35 ++++++++++++++++++++++++++++++++++-
>  src/util/cgroup.h        |    3 +++
>  6 files changed, 115 insertions(+), 9 deletions(-)
> 
> diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
> index 812cf1d..f6fdc66 100644
> --- a/src/libvirt_private.syms
> +++ b/src/libvirt_private.syms
> @@ -79,6 +79,7 @@ virCgroupGetCpuShares;
>  virCgroupGetCpuacctPercpuUsage;
>  virCgroupGetCpuacctStat;
>  virCgroupGetCpuacctUsage;
> +virCgroupGetCpusetCpus;
>  virCgroupGetCpusetMems;
>  virCgroupGetFreezerState;
>  virCgroupGetMemSwapHardLimit;
> @@ -97,6 +98,7 @@ virCgroupSetBlkioWeight;
>  virCgroupSetCpuCfsPeriod;
>  virCgroupSetCpuCfsQuota;
>  virCgroupSetCpuShares;
> +virCgroupSetCpusetCpus;
>  virCgroupSetCpusetMems;
>  virCgroupSetFreezerState;
>  virCgroupSetMemSwapHardLimit;
> diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
> index 5f7e8b0..6c811ce 100644
> --- a/src/qemu/qemu_cgroup.c
> +++ b/src/qemu/qemu_cgroup.c
> @@ -480,11 +480,49 @@ cleanup:
>      return -1;
>  }
>  
> +int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup, virDomainDefPtr def,
> +                           int vcpuid)
> +{
> +    int i, rc = 0;
> +    char *new_cpus = NULL;
> +
> +    if (vcpuid < 0 || vcpuid >= def->vcpus) {
> +        virReportSystemError(EINVAL,
> +                             _("invalid vcpuid: %d"), vcpuid);
> +        return -EINVAL;
> +    }
> +
> +    for (i = 0; i < def->cputune.nvcpupin; i++) {
> +        if (vcpuid == def->cputune.vcpupin[i]->vcpuid) {
> +            new_cpus = virDomainCpuSetFormat(def->cputune.vcpupin[i]->cpumask,
> +                                             VIR_DOMAIN_CPUMASK_LEN);
> +            if (!new_cpus) {
> +                qemuReportError(VIR_ERR_INTERNAL_ERROR,
> +                                _("failed to convert cpu mask"));
> +                rc = -1;
> +                goto cleanup;
> +            }
> +            rc = virCgroupSetCpusetCpus(cgroup, new_cpus);
> +            if (rc != 0) {
> +                virReportSystemError(-rc,
> +                                     "%s",
> +                                     _("Unable to set cpuset.cpus"));
> +                goto cleanup;
> +            }
> +        }
> +    }
> +
> +cleanup:
> +    VIR_FREE(new_cpus);
> +    return rc;
> +}
> +
>  int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
>  {
>      virCgroupPtr cgroup = NULL;
>      virCgroupPtr cgroup_vcpu = NULL;
>      qemuDomainObjPrivatePtr priv = vm->privateData;
> +    virDomainDefPtr def = vm->def;
>      int rc;
>      unsigned int i;
>      unsigned long long period = vm->def->cputune.period;
> @@ -557,6 +595,12 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
>              }
>          }
>  
> +        /* Set vcpupin in cgroup if vcpupin xml is provided */
> +        if (def->cputune.nvcpupin &&
> +            qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET) &&
> +            qemuSetupCgroupVcpuPin(cgroup_vcpu, def, i) < 0)
> +            goto cleanup;
> +
>          virCgroupFree(&cgroup_vcpu);
>      }
>  
> diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
> index cf0d383..91d5632 100644
> --- a/src/qemu/qemu_cgroup.h
> +++ b/src/qemu/qemu_cgroup.h
> @@ -53,6 +53,8 @@ int qemuSetupCgroup(struct qemud_driver *driver,
>  int qemuSetupCgroupVcpuBW(virCgroupPtr cgroup,
>                            unsigned long long period,
>                            long long quota);
> +int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup, virDomainDefPtr def,
> +                           int vcpuid);
>  int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm);
>  int qemuSetupCgroupForHypervisor(struct qemud_driver *driver,
>                                   virDomainObjPtr vm);
> diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
> index 3410535..9f795c1 100644
> --- a/src/qemu/qemu_driver.c
> +++ b/src/qemu/qemu_driver.c
> @@ -3613,6 +3613,8 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
>      struct qemud_driver *driver = dom->conn->privateData;
>      virDomainObjPtr vm;
>      virDomainDefPtr persistentDef = NULL;
> +    virCgroupPtr cgroup_dom = NULL;
> +    virCgroupPtr cgroup_vcpu = NULL;
>      int maxcpu, hostcpus;
>      virNodeInfo nodeinfo;
>      int ret = -1;
> @@ -3667,9 +3669,32 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
>      if (flags & VIR_DOMAIN_AFFECT_LIVE) {
>  
>          if (priv->vcpupids != NULL) {
> +            /* Add config to vm->def first, because cgroup APIs need it. */

s/cgroup APIs need/qemuSetupCgroupVcpuPin needs/

> +            if (virDomainVcpuPinAdd(vm->def, cpumap, maplen, vcpu) < 0) {
> +                qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> +                                _("failed to update or add vcpupin xml of "
> +                                  "a running domain"));
> +                goto cleanup;
> +            }
> +
> +            /* Configure the corresponding cpuset cgroup before set affinity. */
> +            if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET) &&
> +                virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup_dom, 0) == 0 &&
> +                virCgroupForVcpu(cgroup_dom, vcpu, &cgroup_vcpu, 0) == 0 &&
> +                qemuSetupCgroupVcpuPin(cgroup_vcpu, vm->def, vcpu) < 0) {
> +                qemuReportError(VIR_ERR_OPERATION_INVALID,
> +                                _("failed to set cpuset.cpus in cgroup"
> +                                  " for vcpu %d"), vcpu);
> +                goto cleanup;
> +            }

This enforces the presence of cpuset cgroup to support vcpupin despite
the fact that we can do vcpupin by sched_setaffinity(). Can we make
vcpupin work even when there is no cgroup?

> +
>              if (virProcessInfoSetAffinity(priv->vcpupids[vcpu],
> -                                          cpumap, maplen, maxcpu) < 0)
> +                                          cpumap, maplen, maxcpu) < 0) {
> +                qemuReportError(VIR_ERR_SYSTEM_ERROR,

No longer qemuReportError, virReportError instead.

> +                                _("failed to set cpu affinity for vcpu %d"),
> +                                vcpu);
>                  goto cleanup;
> +            }
>          } else {
>              qemuReportError(VIR_ERR_OPERATION_INVALID,
>                              "%s", _("cpu affinity is not supported"));
> @@ -3683,13 +3708,6 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
>                                    "a running domain"));
>                  goto cleanup;
>              }
> -        } else {
> -            if (virDomainVcpuPinAdd(vm->def, cpumap, maplen, vcpu) < 0) {
> -                qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> -                                _("failed to update or add vcpupin xml of "
> -                                  "a running domain"));
> -                goto cleanup;
> -            }
>          }
>  
>          if (virDomainSaveStatus(driver->caps, driver->stateDir, vm) < 0)
> @@ -3721,6 +3739,10 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
>      ret = 0;
>  
>  cleanup:
> +    if (cgroup_vcpu)
> +        virCgroupFree(&cgroup_vcpu);
> +    if (cgroup_dom)
> +        virCgroupFree(&cgroup_dom);
>      if (vm)
>          virDomainObjUnlock(vm);
>      return ret;
> diff --git a/src/util/cgroup.c b/src/util/cgroup.c
> index 3e9ba4e..9ffbaac 100644
> --- a/src/util/cgroup.c
> +++ b/src/util/cgroup.c
> @@ -532,7 +532,8 @@ static int virCgroupMakeGroup(virCgroupPtr parent, virCgroupPtr group,
>          /* We need to control cpu bandwidth for each vcpu now */
>          if ((flags & VIR_CGROUP_VCPU) &&
>              (i != VIR_CGROUP_CONTROLLER_CPU &&
> -             i != VIR_CGROUP_CONTROLLER_CPUACCT)) {
> +             i != VIR_CGROUP_CONTROLLER_CPUACCT &&
> +             i != VIR_CGROUP_CONTROLLER_CPUSET)) {
>              /* treat it as unmounted and we can use virCgroupAddTask */
>              VIR_FREE(group->controllers[i].mountPoint);
>              continue;
> @@ -1409,6 +1410,38 @@ int virCgroupGetCpusetMems(virCgroupPtr group, char **mems)
>  }
>  
>  /**
> + * virCgroupSetCpusetCpus:
> + *
> + * @group: The cgroup to set cpuset.cpus for
> + * @cpus: the cpus to set
> + *
> + * Retuens: 0 on success
> + */
> +int virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus)
> +{
> +    return virCgroupSetValueStr(group,
> +                                VIR_CGROUP_CONTROLLER_CPUSET,
> +                                "cpuset.cpus",
> +                                cpus);
> +}
> +
> +/**
> + * virCgroupGetCpusetCpus:
> + *
> + * @group: The cgroup to get cpuset.cpus for
> + * @cpus: the cpus to get
> + *
> + * Retuens: 0 on success
> + */
> +int virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus)
> +{
> +    return virCgroupGetValueStr(group,
> +                                VIR_CGROUP_CONTROLLER_CPUSET,
> +                                "cpuset.cpus",
> +                                cpus);
> +}
> +
> +/**
>   * virCgroupDenyAllDevices:
>   *
>   * @group: The cgroup to deny all permissions, for all devices
> diff --git a/src/util/cgroup.h b/src/util/cgroup.h
> index f14c167..b196214 100644
> --- a/src/util/cgroup.h
> +++ b/src/util/cgroup.h
> @@ -139,6 +139,9 @@ int virCgroupGetFreezerState(virCgroupPtr group, char **state);
>  int virCgroupSetCpusetMems(virCgroupPtr group, const char *mems);
>  int virCgroupGetCpusetMems(virCgroupPtr group, char **mems);
>  
> +int virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus);
> +int virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus);
> +
>  int virCgroupRemove(virCgroupPtr group);
>  
>  void virCgroupFree(virCgroupPtr *group);
> -- 
> 1.7.10.2
> 
> --
> libvir-list mailing list
> libvir-list redhat com
> https://www.redhat.com/mailman/listinfo/libvir-list

-- 
Thanks,
Hu Tao


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]