[libvirt] [PATCH v4 10/14] qemu: Start PR daemon on domain startup

[Michal Privoznik]

Before we exec() qemu we have to spawn pr-helper processes for
all managed reservations (well, technically there can only one).
The only caveat there is that we should place the process into
the same namespace and cgroup as qemu (so that it shares the same
view of the system). But we can do that only after we've forked.
That means calling the setup function between fork() and exec().

Signed-off-by: Michal Privoznik <mprivozn at redhat.com>
---
 src/qemu/qemu_process.c | 224 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 224 insertions(+)

diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index f02114c693..982c989a0a 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -2556,6 +2556,223 @@ qemuProcessResctrlCreate(virQEMUDriverPtr driver,
     ret = 0;
  cleanup:
     virObjectUnref(caps);
+
+    return ret;
+}
+
+
+static char *
+qemuProcessBuildPRHelperPidfilePath(virDomainObjPtr vm,
+                                    const char *prdAlias)
+{
+    qemuDomainObjPrivatePtr priv = vm->privateData;
+
+    return virPidFileBuildPath(priv->libDir, prdAlias);
+}
+
+
+static void
+qemuProcessKillPRDaemon(virDomainObjPtr vm)
+{
+    virErrorPtr orig_err;
+    char *prAlias;
+    char *prPidfile;
+
+    if (!(prAlias = qemuDomainGetManagedPRAlias())) {
+        VIR_WARN("Unable to get default PR manager alias");
+        return;
+    }
+
+    if (!(prPidfile = qemuProcessBuildPRHelperPidfilePath(vm, prAlias))) {
+        VIR_WARN("Unable to construct pr-helper pidfile path");
+        VIR_FREE(prAlias);
+        return;
+    }
+    VIR_FREE(prAlias);
+
+    virErrorPreserveLast(&orig_err);
+    if (virPidFileForceCleanupPath(prPidfile) < 0) {
+        VIR_WARN("Unable to kill pr-helper process");
+    } else if (unlink(prPidfile) < 0 &&
+               errno != ENOENT) {
+        virReportSystemError(errno,
+                             _("Unable to remove stale pidfile %s"),
+                             prPidfile);
+    }
+    virErrorRestore(&orig_err);
+
+    VIR_FREE(prPidfile);
+}
+
+
+static int
+qemuProcessStartPRDaemonHook(void *opaque)
+{
+    virDomainObjPtr vm = opaque;
+    size_t i, nfds = 0;
+    int *fds = NULL;
+    int ret = -1;
+
+    if (virProcessGetNamespaces(vm->pid, &nfds, &fds) < 0)
+        return ret;
+
+    if (nfds > 0 &&
+        virProcessSetNamespaces(nfds, fds) < 0)
+        goto cleanup;
+
+    ret = 0;
+ cleanup:
+    for (i = 0; i < nfds; i++)
+        VIR_FORCE_CLOSE(fds[i]);
+    VIR_FREE(fds);
+    return ret;
+}
+
+
+static int
+qemuProcessStartPRDaemon(virDomainObjPtr vm,
+                         qemuDomainDiskPRDPtr prd)
+{
+    qemuDomainObjPrivatePtr priv = vm->privateData;
+    virQEMUDriverPtr driver = priv->driver;
+    virQEMUDriverConfigPtr cfg;
+    int errfd = -1;
+    char *pidfile = NULL;
+    int pidfd = -1;
+    pid_t cpid = -1;
+    virCommandPtr cmd = NULL;
+    virTimeBackOffVar timebackoff;
+    const unsigned long long timeout = 500000; /* ms */
+    int ret = -1;
+
+    cfg = virQEMUDriverGetConfig(driver);
+
+    if (!virFileIsExecutable(cfg->prHelperName)) {
+        virReportSystemError(errno, _("'%s' is not a suitable pr helper"),
+                             cfg->prHelperName);
+        goto cleanup;
+    }
+
+    if (!(pidfile = qemuProcessBuildPRHelperPidfilePath(vm, prd->alias)))
+        goto cleanup;
+
+    /* Just try to acquire. Dummy pid will be replaced later */
+    if ((pidfd = virPidFileAcquirePath(pidfile, false, -1)) < 0)
+        goto cleanup;
+
+    /* Remove stale socket */
+    if (unlink(prd->path) < 0 &&
+        errno != ENOENT) {
+        virReportSystemError(errno,
+                             _("Unable to remove stale socket path: %s"),
+                             prd->path);
+        goto cleanup;
+    }
+
+    if (!(cmd = virCommandNewArgList(cfg->prHelperName,
+                                     "-k", prd->path,
+                                     "-f", pidfile,
+                                     NULL)))
+        goto cleanup;
+
+    virCommandDaemonize(cmd);
+    /* We want our virCommand to write child PID into the pidfile
+     * so that we can read it even before exec(). */
+    virCommandSetPidFile(cmd, pidfile);
+    virCommandSetErrorFD(cmd, &errfd);
+
+    /* Place the process into the same namespace and cgroup as
+     * qemu (so that it shares the same view of the system). */
+    virCommandSetPreExecHook(cmd, qemuProcessStartPRDaemonHook, vm);
+
+    if (virCommandRun(cmd, NULL) < 0)
+        goto cleanup;
+
+    if (virPidFileReadPath(pidfile, &cpid) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("pr helper %s didn't show up"), prd->alias);
+        goto cleanup;
+    }
+
+    if (virTimeBackOffStart(&timebackoff, 1, timeout) < 0)
+        goto cleanup;
+    while (virTimeBackOffWait(&timebackoff)) {
+        char errbuf[1024] = { 0 };
+
+        if (virFileExists(prd->path))
+            break;
+
+        if (virProcessKill(cpid, 0) == 0)
+            continue;
+
+        if (saferead(errfd, errbuf, sizeof(errbuf) - 1) < 0) {
+            virReportSystemError(errno,
+                                 _("pr helper %s died unexpectedly"), prd->alias);
+        } else {
+            virReportError(VIR_ERR_OPERATION_FAILED,
+                           _("pr helper died and reported: %s"), errbuf);
+        }
+        goto cleanup;
+    }
+
+    if (!virFileExists(prd->path)) {
+        virReportError(VIR_ERR_OPERATION_TIMEOUT, "%s",
+                       _("pr helper socked did not show up"));
+        goto cleanup;
+    }
+
+    if (priv->cgroup &&
+        virCgroupAddMachineTask(priv->cgroup, cpid) < 0)
+        goto cleanup;
+
+    if (qemuSecurityDomainSetPathLabel(driver->securityManager,
+                                       vm->def, prd->path, true) < 0)
+        goto cleanup;
+
+    ret = 0;
+ cleanup:
+    if (ret < 0) {
+        virCommandAbort(cmd);
+        if (cpid >= 0)
+            virProcessKillPainfully(cpid, true);
+        unlink(pidfile);
+    }
+    virCommandFree(cmd);
+    VIR_FORCE_CLOSE(pidfd);
+    VIR_FREE(pidfile);
+    VIR_FORCE_CLOSE(errfd);
+    virObjectUnref(cfg);
+    return ret;
+}
+
+
+static int
+qemuProcessMaybeStartPRDaemon(virDomainObjPtr vm)
+{
+    size_t i;
+    int ret = -1;
+    qemuDomainDiskPRDPtr prd = NULL;
+
+    for (i = 0; i < vm->def->ndisks; i++) {
+        const virDomainDiskDef *disk = vm->def->disks[i];
+        qemuDomainStorageSourcePrivatePtr srcPriv;
+
+        if (!virStoragePRDefIsManaged(disk->src->pr))
+            continue;
+
+        srcPriv = QEMU_DOMAIN_STORAGE_SOURCE_PRIVATE(disk->src);
+        prd = srcPriv->prd;
+        break;
+    }
+
+    if (prd &&
+        qemuProcessStartPRDaemon(vm, prd) < 0)
+        goto cleanup;
+
+    ret = 0;
+ cleanup:
+    if (ret < 0)
+        qemuProcessKillPRDaemon(vm);
     return ret;
 }
 
@@ -6073,6 +6290,10 @@ qemuProcessLaunch(virConnectPtr conn,
     if (qemuProcessResctrlCreate(driver, vm) < 0)
         goto cleanup;
 
+    VIR_DEBUG("Setting up PR daemon");
+    if (qemuProcessMaybeStartPRDaemon(vm) < 0)
+        goto cleanup;
+
     VIR_DEBUG("Setting domain security labels");
     if (qemuSecuritySetAllLabel(driver,
                                 vm,
@@ -6600,6 +6821,9 @@ void qemuProcessStop(virQEMUDriverPtr driver,
     /* Remove the master key */
     qemuDomainMasterKeyRemove(priv);
 
+    /* Do this before we delete the tree and remove pidfile. */
+    qemuProcessKillPRDaemon(vm);
+
     virFileDeleteTree(priv->libDir);
     virFileDeleteTree(priv->channelTargetDir);
 
-- 
2.16.1