[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [libvirt] [PATCH v3] qemu: Process RDMA GID state change event



On Thu, Nov 29, 2018 at 11:11:48PM +0200, Yuval Shaia wrote:
> This event is emitted on the monitor when a GID table in pvrdma device
> is modified and the change needs to be propagate to the backend RDMA
> device's GID table.
> 
> The control over the RDMA device's GID table is done by updating the
> device's Ethernet function addresses.
> Usually the first GID entry is determine by the MAC address, the second
> by the first IPv6 address and the third by the IPv4 address. Other
> entries can be added by adding more IP addresses. The opposite is the
> same, i.e. whenever an address is removed, the corresponding GID entry
> is removed.
> 
> The process is done by the network and RDMA stacks. Whenever an address
> is added the ib_core driver is notified and calls the device driver's
> add_gid function which in turn update the device.
> 
> To support this in pvrdma device we need to hook into the create_bind
> and destroy_bind HW commands triggered by pvrdma driver in guest.
> Whenever a changed is made to the pvrdma device's GID table a special
> QMP messages is sent to be processed by libvirt to update the address of
> the backend Ethernet device.
> 
> Signed-off-by: Yuval Shaia <yuval shaia oracle com>
> ---
> v1 -> v2:
> 	* Address all comments from Michal Privoznik
> 
> v2 -> v3:
> 	* Remove static initialization in processRdmaGidStatusChangedEvent

Hi,
Relevant qemu patch-set was applied to master.
(https://www.mail-archive.com/qemu-devel nongnu org/msg583387.html)

Appreciate if this can be reviewed and applied as well.

Thanks,
Yuval

> ---
>  src/qemu/qemu_domain.c       |  3 +++
>  src/qemu/qemu_domain.h       |  1 +
>  src/qemu/qemu_driver.c       | 44 ++++++++++++++++++++++++++++++
>  src/qemu/qemu_monitor.c      | 27 +++++++++++++++++++
>  src/qemu/qemu_monitor.h      | 27 +++++++++++++++++++
>  src/qemu/qemu_monitor_json.c | 36 +++++++++++++++++++++++++
>  src/qemu/qemu_process.c      | 52 ++++++++++++++++++++++++++++++++++++
>  7 files changed, 190 insertions(+)
> 
> diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
> index ba3fff607a..8da54c7ee9 100644
> --- a/src/qemu/qemu_domain.c
> +++ b/src/qemu/qemu_domain.c
> @@ -13479,6 +13479,9 @@ qemuProcessEventFree(struct qemuProcessEvent *event)
>      case QEMU_PROCESS_EVENT_GUESTPANIC:
>          qemuMonitorEventPanicInfoFree(event->data);
>          break;
> +    case QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED:
> +        qemuMonitorEventRdmaGidStatusFree(event->data);
> +        break;
>      case QEMU_PROCESS_EVENT_WATCHDOG:
>      case QEMU_PROCESS_EVENT_DEVICE_DELETED:
>      case QEMU_PROCESS_EVENT_NIC_RX_FILTER_CHANGED:
> diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
> index 80bd4bde91..64bceb9a98 100644
> --- a/src/qemu/qemu_domain.h
> +++ b/src/qemu/qemu_domain.h
> @@ -487,6 +487,7 @@ typedef enum {
>      QEMU_PROCESS_EVENT_BLOCK_JOB,
>      QEMU_PROCESS_EVENT_MONITOR_EOF,
>      QEMU_PROCESS_EVENT_PR_DISCONNECT,
> +    QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED,
>  
>      QEMU_PROCESS_EVENT_LAST
>  } qemuProcessEventType;
> diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
> index a52e2495d5..5c6ab3c0ea 100644
> --- a/src/qemu/qemu_driver.c
> +++ b/src/qemu/qemu_driver.c
> @@ -4788,6 +4788,47 @@ processPRDisconnectEvent(virDomainObjPtr vm)
>  }
>  
>  
> +static void
> +processRdmaGidStatusChangedEvent(virDomainObjPtr vm,
> +                                 qemuMonitorRdmaGidStatusChangedPrivatePtr info)
> +{
> +    unsigned int prefix_len;
> +    virSocketAddr addr;
> +    int rc;
> +
> +    if (!virDomainObjIsActive(vm))
> +        return;
> +
> +    VIR_DEBUG("netdev=%s,gid_status=%d,subnet_prefix=0x%llx,interface_id=0x%llx",
> +              info->netdev, info->gid_status, info->subnet_prefix,
> +              info->interface_id);
> +
> +    if (info->subnet_prefix) {
> +        prefix_len = 64;
> +        uint32_t ipv6[4];
> +        memcpy(&ipv6[0], &info->subnet_prefix, sizeof(info->subnet_prefix));
> +        memcpy(&ipv6[2], &info->interface_id, sizeof(info->interface_id));
> +        virSocketAddrSetIPv6AddrNetOrder(&addr, ipv6);
> +    } else {
> +        prefix_len = 24;
> +        virSocketAddrSetIPv4AddrNetOrder(&addr, info->interface_id >> 32);
> +    }
> +
> +    if (info->gid_status) {
> +        VIR_DEBUG("Adding %s to %s", virSocketAddrFormat(&addr), info->netdev);
> +        rc = virNetDevIPAddrAdd(info->netdev, &addr, NULL, prefix_len);
> +    } else {
> +        VIR_DEBUG("Removing %s from %s", virSocketAddrFormat(&addr),
> +                  info->netdev);
> +        rc = virNetDevIPAddrDel(info->netdev, &addr, prefix_len);
> +    }
> +
> +    if (rc < 0)
> +        VIR_WARN("Fail to update address %s to %s", virSocketAddrFormat(&addr),
> +                 info->netdev);
> +}
> +
> +
>  static void qemuProcessEventHandler(void *data, void *opaque)
>  {
>      struct qemuProcessEvent *processEvent = data;
> @@ -4828,6 +4869,9 @@ static void qemuProcessEventHandler(void *data, void *opaque)
>      case QEMU_PROCESS_EVENT_PR_DISCONNECT:
>          processPRDisconnectEvent(vm);
>          break;
> +    case QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED:
> +        processRdmaGidStatusChangedEvent(vm, processEvent->data);
> +        break;
>      case QEMU_PROCESS_EVENT_LAST:
>          break;
>      }
> diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
> index 7f7013e115..4bf71dbf8c 100644
> --- a/src/qemu/qemu_monitor.c
> +++ b/src/qemu/qemu_monitor.c
> @@ -1686,6 +1686,22 @@ qemuMonitorEmitPRManagerStatusChanged(qemuMonitorPtr mon,
>  }
>  
>  
> +int
> +qemuMonitorEmitRdmaGidStatusChanged(qemuMonitorPtr mon, const char *netdev,
> +                                    bool gid_status, uint64_t subnet_prefix,
> +                                    uint64_t interface_id)
> +{
> +    int ret = -1;
> +    VIR_DEBUG("netdev=%s,gid_status=%d,subnet_prefix=0x%lx,interface_id=0x%lx",
> +              netdev, gid_status, subnet_prefix, interface_id);
> +
> +    QEMU_MONITOR_CALLBACK(mon, ret, domainRdmaGidStatusChanged, mon->vm, netdev,
> +                          gid_status, subnet_prefix, interface_id);
> +
> +    return ret;
> +}
> +
> +
>  int
>  qemuMonitorSetCapabilities(qemuMonitorPtr mon)
>  {
> @@ -4298,6 +4314,17 @@ qemuMonitorEventPanicInfoFree(qemuMonitorEventPanicInfoPtr info)
>  }
>  
>  
> +void
> +qemuMonitorEventRdmaGidStatusFree(qemuMonitorRdmaGidStatusChangedPrivatePtr info)
> +{
> +    if (!info)
> +        return;
> +
> +    VIR_FREE(info->netdev);
> +    VIR_FREE(info);
> +}
> +
> +
>  int
>  qemuMonitorSetWatchdogAction(qemuMonitorPtr mon,
>                               const char *action)
> diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
> index 48b142a4f4..b639a0a9d2 100644
> --- a/src/qemu/qemu_monitor.h
> +++ b/src/qemu/qemu_monitor.h
> @@ -109,8 +109,22 @@ struct _qemuMonitorEventPanicInfo {
>      } data;
>  };
>  
> +
> +typedef struct _qemuMonitorRdmaGidStatusChangedPrivate qemuMonitorRdmaGidStatusChangedPrivate;
> +typedef qemuMonitorRdmaGidStatusChangedPrivate *qemuMonitorRdmaGidStatusChangedPrivatePtr;
> +struct _qemuMonitorRdmaGidStatusChangedPrivate {
> +    virObject parent;
> +
> +    char *netdev;
> +    bool gid_status;
> +    unsigned long long subnet_prefix;
> +    unsigned long long interface_id;
> +};
> +
> +
>  char *qemuMonitorGuestPanicEventInfoFormatMsg(qemuMonitorEventPanicInfoPtr info);
>  void qemuMonitorEventPanicInfoFree(qemuMonitorEventPanicInfoPtr info);
> +void qemuMonitorEventRdmaGidStatusFree(qemuMonitorRdmaGidStatusChangedPrivatePtr info);
>  
>  typedef void (*qemuMonitorDestroyCallback)(qemuMonitorPtr mon,
>                                             virDomainObjPtr vm,
> @@ -281,6 +295,14 @@ typedef int (*qemuMonitorDomainPRManagerStatusChangedCallback)(qemuMonitorPtr mo
>                                                                 bool connected,
>                                                                 void *opaque);
>  
> +typedef int (*qemuMonitorDomainRdmaGidStatusChangedCallback)(qemuMonitorPtr mon,
> +                                                             virDomainObjPtr vm,
> +                                                             const char *netdev,
> +                                                             bool gid_status,
> +                                                             uint64_t subnet_prefix,
> +                                                             uint64_t interface_id,
> +                                                             void *opaque);
> +
>  typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks;
>  typedef qemuMonitorCallbacks *qemuMonitorCallbacksPtr;
>  struct _qemuMonitorCallbacks {
> @@ -314,6 +336,7 @@ struct _qemuMonitorCallbacks {
>      qemuMonitorDomainBlockThresholdCallback domainBlockThreshold;
>      qemuMonitorDomainDumpCompletedCallback domainDumpCompleted;
>      qemuMonitorDomainPRManagerStatusChangedCallback domainPRManagerStatusChanged;
> +    qemuMonitorDomainRdmaGidStatusChangedCallback domainRdmaGidStatusChanged;
>  };
>  
>  char *qemuMonitorEscapeArg(const char *in);
> @@ -448,6 +471,10 @@ int qemuMonitorEmitPRManagerStatusChanged(qemuMonitorPtr mon,
>                                            const char *prManager,
>                                            bool connected);
>  
> +int qemuMonitorEmitRdmaGidStatusChanged(qemuMonitorPtr mon, const char *netdev,
> +                                        bool gid_status, uint64_t subnet_prefix,
> +                                        uint64_t interface_id);
> +
>  int qemuMonitorStartCPUs(qemuMonitorPtr mon);
>  int qemuMonitorStopCPUs(qemuMonitorPtr mon);
>  
> diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c
> index 3de298c9e2..8df9b426ba 100644
> --- a/src/qemu/qemu_monitor_json.c
> +++ b/src/qemu/qemu_monitor_json.c
> @@ -91,6 +91,7 @@ static void qemuMonitorJSONHandleAcpiOstInfo(qemuMonitorPtr mon, virJSONValuePtr
>  static void qemuMonitorJSONHandleBlockThreshold(qemuMonitorPtr mon, virJSONValuePtr data);
>  static void qemuMonitorJSONHandleDumpCompleted(qemuMonitorPtr mon, virJSONValuePtr data);
>  static void qemuMonitorJSONHandlePRManagerStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data);
> +static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data);
>  
>  typedef struct {
>      const char *type;
> @@ -114,6 +115,7 @@ static qemuEventHandler eventHandlers[] = {
>      { "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, },
>      { "POWERDOWN", qemuMonitorJSONHandlePowerdown, },
>      { "PR_MANAGER_STATUS_CHANGED", qemuMonitorJSONHandlePRManagerStatusChanged, },
> +    { "RDMA_GID_STATUS_CHANGED", qemuMonitorJSONHandleRdmaGidStatusChanged, },
>      { "RESET", qemuMonitorJSONHandleReset, },
>      { "RESUME", qemuMonitorJSONHandleResume, },
>      { "RTC_CHANGE", qemuMonitorJSONHandleRTCChange, },
> @@ -1351,6 +1353,40 @@ static void qemuMonitorJSONHandlePRManagerStatusChanged(qemuMonitorPtr mon,
>  }
>  
>  
> +static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitorPtr mon,
> +                                                      virJSONValuePtr data)
> +{
> +    const char *netdev;
> +    bool gid_status;
> +    unsigned long long subnet_prefix, interface_id;
> +
> +    if (!(netdev = virJSONValueObjectGetString(data, "netdev"))) {
> +        VIR_WARN("missing netdev in GID_STATUS_CHANGED event");
> +        return;
> +    }
> +
> +    if (virJSONValueObjectGetBoolean(data, "gid-status", &gid_status)) {
> +        VIR_WARN("missing gid-status in GID_STATUS_CHANGED event");
> +        return;
> +    }
> +
> +    if (virJSONValueObjectGetNumberUlong(data, "subnet-prefix",
> +                                         &subnet_prefix)) {
> +        VIR_WARN("missing subnet-prefix in GID_STATUS_CHANGED event");
> +        return;
> +    }
> +
> +    if (virJSONValueObjectGetNumberUlong(data, "interface-id",
> +                                         &interface_id)) {
> +        VIR_WARN("missing interface-id in GID_STATUS_CHANGED event");
> +        return;
> +    }
> +
> +    qemuMonitorEmitRdmaGidStatusChanged(mon, netdev, gid_status, subnet_prefix,
> +                                        interface_id);
> +}
> +
> +
>  int
>  qemuMonitorJSONHumanCommandWithFd(qemuMonitorPtr mon,
>                                    const char *cmd_str,
> diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
> index 9cf971808c..6cf0ace5cf 100644
> --- a/src/qemu/qemu_process.c
> +++ b/src/qemu/qemu_process.c
> @@ -1703,6 +1703,57 @@ qemuProcessHandlePRManagerStatusChanged(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
>  }
>  
>  
> +static int
> +qemuProcessHandleRdmaGidStatusChanged(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
> +                                      virDomainObjPtr vm, const char *netdev,
> +                                      bool gid_status, uint64_t subnet_prefix,
> +                                      uint64_t interface_id, void *opaque)
> +{
> +    virQEMUDriverPtr driver = opaque;
> +    struct qemuProcessEvent *processEvent = NULL;
> +    qemuMonitorRdmaGidStatusChangedPrivatePtr rdmaGitStatusChangedPriv = NULL;
> +    int ret = -1;
> +
> +    virObjectLock(vm);
> +
> +    VIR_DEBUG("netdev=%s,gid_status=%d,subnet_prefix=0x%lx,interface_id=0x%lx",
> +              netdev, gid_status, subnet_prefix, interface_id);
> +
> +    if (VIR_ALLOC(rdmaGitStatusChangedPriv) < 0)
> +        goto out_unlock;
> +
> +    if (VIR_STRDUP(rdmaGitStatusChangedPriv->netdev, netdev) < 0)
> +        goto out_free;
> +
> +    rdmaGitStatusChangedPriv->gid_status = gid_status;
> +    rdmaGitStatusChangedPriv->subnet_prefix = subnet_prefix;
> +    rdmaGitStatusChangedPriv->interface_id = interface_id;
> +
> +    if (VIR_ALLOC(processEvent) < 0)
> +        goto out_free;
> +
> +    processEvent->eventType = QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED;
> +    processEvent->vm = virObjectRef(vm);
> +    processEvent->data = rdmaGitStatusChangedPriv;
> +
> +    if (virThreadPoolSendJob(driver->workerPool, 0, processEvent) < 0) {
> +        qemuProcessEventFree(processEvent);
> +        virObjectUnref(vm);
> +        goto out_free;
> +    }
> +
> +    ret = 0;
> +    goto out_unlock;
> +
> + out_free:
> +    qemuMonitorEventRdmaGidStatusFree(rdmaGitStatusChangedPriv);
> +
> + out_unlock:
> +    virObjectUnlock(vm);
> +    return ret;
> +}
> +
> +
>  static qemuMonitorCallbacks monitorCallbacks = {
>      .eofNotify = qemuProcessHandleMonitorEOF,
>      .errorNotify = qemuProcessHandleMonitorError,
> @@ -1732,6 +1783,7 @@ static qemuMonitorCallbacks monitorCallbacks = {
>      .domainBlockThreshold = qemuProcessHandleBlockThreshold,
>      .domainDumpCompleted = qemuProcessHandleDumpCompleted,
>      .domainPRManagerStatusChanged = qemuProcessHandlePRManagerStatusChanged,
> +    .domainRdmaGidStatusChanged = qemuProcessHandleRdmaGidStatusChanged,
>  };
>  
>  static void
> -- 
> 2.17.2
> 


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]