[libvirt] [PATCHv2 05/11] use cpuset to manage numa

[Daniel P. Berrange]

On Thu, Nov 17, 2011 at 05:44:15PM +0800, Hu Tao wrote:
> This patch deprecates libnuma and uses cpuset to manage numa.
> We can further add a `numatune' command to adjust numa parameters
> through cpuset.
> ---
>  src/qemu/qemu_cgroup.c |   73 ++++++++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 73 insertions(+), 0 deletions(-)
> 
> diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
> index 2a10bd2..9dd67b7 100644
> --- a/src/qemu/qemu_cgroup.c
> +++ b/src/qemu/qemu_cgroup.c
> @@ -365,6 +365,79 @@ int qemuSetupCgroup(struct qemud_driver *driver,
>          }
>      }
>  
> +    if (vm->def->numatune.memory.nodemask &&
> +        vm->def->numatune.backend == VIR_DOMAIN_NUMATUNE_BACKEND_NONE) {
> +        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) {
> +            int mode = vm->def->numatune.memory.mode;
> +            char *mask = virDomainCpuSetFormat(vm->def->numatune.memory.nodemask, VIR_DOMAIN_CPUMASK_LEN);
> +            if (!mask) {
> +                qemuReportError(VIR_ERR_INTERNAL_ERROR,
> +                                _("failed to convert memory nodemask"));
> +                goto cleanup;
> +            }
> +
> +            rc = virCgroupSetCpusetMems(cgroup, mask);
> +            if (rc != 0) {
> +                virReportSystemError(-rc,
> +                                     _("Unable to set cpuset.mems for domain %s"),
> +                                     vm->def->name);
> +                goto cleanup;
> +            }
> +            switch (mode) {
> +            case VIR_DOMAIN_NUMATUNE_MEM_STRICT:
> +                rc = virCgroupSetCpusetHardwall(cgroup, 1);
> +                if (rc != 0) {
> +                    virReportSystemError(-rc,
> +                                         _("Unable to set cpuset.mem_hardwall for domain %s"),
> +                                         vm->def->name);
> +                    goto cleanup;
> +                }
> +                break;
> +            case VIR_DOMAIN_NUMATUNE_MEM_PREFERRED:
> +                rc = virCgroupSetCpusetMemorySpreadPage(cgroup, 0);
> +                if (rc != 0) {
> +                    virReportSystemError(-rc,
> +                                         _("Unable to set cpuset.memory_spread_page for domain %s"),
> +                                         vm->def->name);
> +                    goto cleanup;
> +                }
> +                rc = virCgroupSetCpusetMemorySpreadSlab(cgroup, 0);
> +                if (rc != 0) {
> +                    virReportSystemError(-rc,
> +                                         _("Unable to set cpuset.memory_spread_slab for domain %s"),
> +                                         vm->def->name);
> +                    goto cleanup;
> +                }
> +                break;
> +            case VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE:
> +                rc = virCgroupSetCpusetMemorySpreadPage(cgroup, 1);
> +                if (rc != 0) {
> +                    virReportSystemError(-rc,
> +                                         _("Unable to set cpuset.memory_spread_page for domain %s"),
> +                                         vm->def->name);
> +                    goto cleanup;
> +                }
> +                rc = virCgroupSetCpusetMemorySpreadSlab(cgroup, 1);
> +                if (rc != 0) {
> +                    virReportSystemError(-rc,
> +                                         _("Unable to set cpuset.memory_spread_slab for domain %s"),
> +                                         vm->def->name);
> +                    goto cleanup;
> +                }
> +                break;
> +            default:
> +                qemuReportError(VIR_ERR_XML_ERROR,
> +                                "%s", _("Invalid mode for memory NUMA tuning."));
> +                goto cleanup;
> +            }
> +            vm->def->numatune.backend = VIR_DOMAIN_NUMATUNE_BACKEND_CGROUP_CPUSET;
> +        } else {
> +            qemuReportError(VIR_ERR_CONFIG_UNSUPPORTED,
> +                            _("Unable to set numa for domain %s since cpuset cgroup is "
> +                              "not available on this host"), vm->def->name);
> +            goto cleanup;
> +        }
> +    }
>  done:
>      virCgroupFree(&cgroup);
>      return 0;

As per my initial comments this is all wrong because the semantics
of the cpuset tunables don't match the policies. IMHO this should
look more like

    if (vm->def->numatune.memory.nodemask &&
        vm->def->numatune.memory.mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
        qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) {
        char *mask = virDomainCpuSetFormat(vm->def->numatune.memory.nodemask, VIR_DOMAIN_CPUMASK_LEN);
        if (!mask) {
            qemuReportError(VIR_ERR_INTERNAL_ERROR,
                            _("failed to convert memory nodemask"));
            goto cleanup;
        }

        rc = virCgroupSetCpusetMems(cgroup, mask);
        VIR_FREE(mask);
        if (rc != 0) {
             virReportSystemError(-rc,
                                  _("Unable to set cpuset.mems for domain %s"),
                                  vm->def->name);
             goto cleanup;
        }
    }

Regards,
Daniel
-- 
|: http://berrange.com      -o-    http://www.flickr.com/photos/dberrange/ :|
|: http://libvirt.org              -o-             http://virt-manager.org :|
|: http://autobuild.org       -o-         http://search.cpan.org/~danberr/ :|
|: http://entangle-photo.org       -o-       http://live.gnome.org/gtk-vnc :|