[libvirt] [PATCH v2] qemu: Set reasonable RSS limit on domain startup

[Michal Privoznik]

If there's a memory leak in qemu or qemu is exploited the host's
system will sooner or later start trashing instead of killing
the bad process. This however has impact on performance and other
guests as well. Therefore we should set a reasonable RSS limit
even when user hasn't set any. It's better to be secure by default.
---

diff to v1:
-the expression for reasonable limit adapted to VRAM + XBZRLE
 (libvirt doesn't override migrate_cache_size yet, so the default
  64MB should fit, and it did indeed during my testing)

 src/qemu/qemu_cgroup.c |   76 +++++++++++++++++++++++++++--------------------
 1 files changed, 44 insertions(+), 32 deletions(-)

diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index dee9de6..b11d28b 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -339,42 +339,54 @@ int qemuSetupCgroup(struct qemud_driver *driver,
         }
     }
 
-    if (vm->def->mem.hard_limit != 0 ||
-        vm->def->mem.soft_limit != 0 ||
-        vm->def->mem.swap_hard_limit != 0) {
-        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_MEMORY)) {
-            if (vm->def->mem.hard_limit != 0) {
-                rc = virCgroupSetMemoryHardLimit(cgroup, vm->def->mem.hard_limit);
-                if (rc != 0) {
-                    virReportSystemError(-rc,
-                                         _("Unable to set memory hard limit for domain %s"),
-                                         vm->def->name);
-                    goto cleanup;
-                }
-            }
-            if (vm->def->mem.soft_limit != 0) {
-                rc = virCgroupSetMemorySoftLimit(cgroup, vm->def->mem.soft_limit);
-                if (rc != 0) {
-                    virReportSystemError(-rc,
-                                         _("Unable to set memory soft limit for domain %s"),
-                                         vm->def->name);
-                    goto cleanup;
-                }
+    if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_MEMORY)) {
+        unsigned long long hard_limit = vm->def->mem.hard_limit;
+
+        if (!hard_limit) {
+            /* If there is no hard_limit set, set a reasonable
+             * one to avoid system trashing caused by exploited qemu.
+             * As 'reasonable limit' has been chosen:
+             *     (1 + k) * (domain memory + total video memory) + F
+             * where k = 0.02 and F = 200MB. */
+            hard_limit = vm->def->mem.max_balloon;
+            for (i = 0; i < vm->def->nvideos; i++)
+                hard_limit += vm->def->videos[i]->vram;
+            hard_limit = hard_limit * 1.02 + 204800;
+        }
+
+        rc = virCgroupSetMemoryHardLimit(cgroup, hard_limit);
+        if (rc != 0) {
+            virReportSystemError(-rc,
+                                 _("Unable to set memory hard limit for domain %s"),
+                                 vm->def->name);
+            goto cleanup;
+        }
+        if (vm->def->mem.soft_limit != 0) {
+            rc = virCgroupSetMemorySoftLimit(cgroup, vm->def->mem.soft_limit);
+            if (rc != 0) {
+                virReportSystemError(-rc,
+                                     _("Unable to set memory soft limit for domain %s"),
+                                     vm->def->name);
+                goto cleanup;
             }
+        }
 
-            if (vm->def->mem.swap_hard_limit != 0) {
-                rc = virCgroupSetMemSwapHardLimit(cgroup, vm->def->mem.swap_hard_limit);
-                if (rc != 0) {
-                    virReportSystemError(-rc,
-                                         _("Unable to set swap hard limit for domain %s"),
-                                         vm->def->name);
-                    goto cleanup;
-                }
+        if (vm->def->mem.swap_hard_limit != 0) {
+            rc = virCgroupSetMemSwapHardLimit(cgroup, vm->def->mem.swap_hard_limit);
+            if (rc != 0) {
+                virReportSystemError(-rc,
+                                     _("Unable to set swap hard limit for domain %s"),
+                                     vm->def->name);
+                goto cleanup;
             }
-        } else {
-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                           _("Memory cgroup is not available on this host"));
         }
+    } else if (vm->def->mem.hard_limit != 0 ||
+               vm->def->mem.soft_limit != 0 ||
+               vm->def->mem.swap_hard_limit != 0) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("Memory cgroup is not available on this host"));
+    } else {
+        VIR_WARN("Could not autoset a RSS limit for domain %s", vm->def->name);
     }
 
     if (vm->def->cputune.shares != 0) {
-- 
1.7.8.6