[libvirt] [RFC PATCH 11/28] qemu: address: Separate the slots into multiple aggregates
Shivaprasad G Bhat
sbhat at linux.vnet.ibm.com
Wed Mar 14 17:19:35 UTC 2018
Today's aggregate flag with the slot being true for pcie-root-ports is not
enough as there will more number of aggregates depending on the number of
Multifuntion PCI cards assigned to the domain.
The aggregate is changed to unsigned int and Zero means Not Applicable, 1 is
reserved for the pcie-root-ports and >= 2 for the the PCI multifunction
cards(coming..).
Signed-off-by: Shivaprasad G Bhat <sbhat at linux.vnet.ibm.com>
---
src/conf/device_conf.h | 1
src/conf/domain_addr.c | 43 ++++++++++++++-------
src/conf/domain_addr.h | 36 ++++++++---------
src/qemu/qemu_domain_address.c | 83 +++++++++++++++++++++++++++++++++-------
src/qemu/qemu_domain_address.h | 8 ++++
5 files changed, 123 insertions(+), 48 deletions(-)
diff --git a/src/conf/device_conf.h b/src/conf/device_conf.h
index f87d6f1fc6..cdb2040fb8 100644
--- a/src/conf/device_conf.h
+++ b/src/conf/device_conf.h
@@ -163,6 +163,7 @@ struct _virDomainDeviceInfo {
* assignment, never saved and never reported.
*/
int pciConnectFlags; /* enum virDomainPCIConnectFlags */
+ unsigned int aggregateSlotIdx; /* Used when the aggregate flag is set */
char *loadparm;
/* PCI devices will only be automatically placed on a PCI bus
diff --git a/src/conf/domain_addr.c b/src/conf/domain_addr.c
index 0c914fe25c..c4a0b99628 100644
--- a/src/conf/domain_addr.c
+++ b/src/conf/domain_addr.c
@@ -63,7 +63,7 @@ virDomainPCIControllerModelToConnectType(virDomainControllerModelPCI model)
return VIR_PCI_CONNECT_TYPE_DMI_TO_PCI_BRIDGE;
case VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT_PORT:
- return VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT | VIR_PCI_CONNECT_AGGREGATE_SLOT;
+ return VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT;
case VIR_DOMAIN_CONTROLLER_MODEL_PCIE_SWITCH_UPSTREAM_PORT:
return VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_UPSTREAM_PORT;
@@ -565,6 +565,7 @@ virDomainPCIAddressReserveAddrInternal(virDomainPCIAddressSetPtr addrs,
virPCIDeviceAddressPtr addr,
virDomainPCIConnectFlags flags,
unsigned int isolationGroup,
+ unsigned int aggregateSlotIdx,
bool fromConfig)
{
int ret = -1;
@@ -598,8 +599,13 @@ virDomainPCIAddressReserveAddrInternal(virDomainPCIAddressSetPtr addrs,
* slot, set the slot's aggregate flag.
*/
if (!bus->slot[addr->slot].functions &&
- flags & VIR_PCI_CONNECT_AGGREGATE_SLOT) {
- bus->slot[addr->slot].aggregate = true;
+ aggregateSlotIdx > 0) {
+ bus->slot[addr->slot].aggregateSlotIdx = aggregateSlotIdx;
+ } else if (bus->slot[addr->slot].aggregateSlotIdx != aggregateSlotIdx && fromConfig) {
+ bus->slot[addr->slot].aggregateSlotIdx = 0;
+ VIR_DEBUG("PCI functions of %.4x:%.2x is aggregated to slot %u"
+ "because of user assigned address %s",
+ addr->domain, addr->bus, aggregateSlotIdx, addrStr);
}
if (virDomainPCIAddressBusIsEmpty(bus) && !bus->isolationGroupLocked) {
@@ -624,8 +630,8 @@ virDomainPCIAddressReserveAddrInternal(virDomainPCIAddressSetPtr addrs,
/* mark the requested function as reserved */
bus->slot[addr->slot].functions |= (1 << addr->function);
- VIR_DEBUG("Reserving PCI address %s (aggregate='%s')", addrStr,
- bus->slot[addr->slot].aggregate ? "true" : "false");
+ VIR_DEBUG("Reserving PCI address %s (aggregateSlotIdx='%d')", addrStr,
+ bus->slot[addr->slot].aggregateSlotIdx);
ret = 0;
cleanup:
@@ -638,10 +644,11 @@ int
virDomainPCIAddressReserveAddr(virDomainPCIAddressSetPtr addrs,
virPCIDeviceAddressPtr addr,
virDomainPCIConnectFlags flags,
- unsigned int isolationGroup)
+ unsigned int isolationGroup,
+ unsigned int aggregateSlotIdx)
{
return virDomainPCIAddressReserveAddrInternal(addrs, addr, flags,
- isolationGroup, true);
+ isolationGroup, aggregateSlotIdx, true);
}
int
@@ -678,6 +685,7 @@ virDomainPCIAddressEnsureAddr(virDomainPCIAddressSetPtr addrs,
ret = virDomainPCIAddressReserveAddrInternal(addrs, &dev->addr.pci,
flags, dev->isolationGroup,
+ dev->aggregateSlotIdx,
true);
} else {
ret = virDomainPCIAddressReserveNextAddr(addrs, dev, flags, -1);
@@ -730,6 +738,7 @@ virDomainPCIAddressSetFree(virDomainPCIAddressSetPtr addrs)
static int
virDomainPCIAddressFindUnusedFunctionOnBus(virDomainPCIAddressBusPtr bus,
virPCIDeviceAddressPtr searchAddr,
+ unsigned int aggregateSlotIdx,
int function,
virDomainPCIConnectFlags flags,
bool *found)
@@ -753,8 +762,8 @@ virDomainPCIAddressFindUnusedFunctionOnBus(virDomainPCIAddressBusPtr bus,
break;
}
- if (flags & VIR_PCI_CONNECT_AGGREGATE_SLOT &&
- bus->slot[searchAddr->slot].aggregate) {
+ if (bus->slot[searchAddr->slot].aggregateSlotIdx > 0 &&
+ bus->slot[searchAddr->slot].aggregateSlotIdx == aggregateSlotIdx) {
/* slot and device are okay with aggregating devices */
if ((bus->slot[searchAddr->slot].functions &
(1 << searchAddr->function)) == 0) {
@@ -799,6 +808,7 @@ virDomainPCIAddressGetNextAddr(virDomainPCIAddressSetPtr addrs,
virPCIDeviceAddressPtr next_addr,
virDomainPCIConnectFlags flags,
unsigned int isolationGroup,
+ unsigned int aggregateSlotIdx,
int function)
{
virPCIDeviceAddress a = { 0 };
@@ -827,7 +837,9 @@ virDomainPCIAddressGetNextAddr(virDomainPCIAddressSetPtr addrs,
a.slot = bus->minSlot;
- if (virDomainPCIAddressFindUnusedFunctionOnBus(bus, &a, function,
+ if (virDomainPCIAddressFindUnusedFunctionOnBus(bus, &a,
+ aggregateSlotIdx,
+ function,
flags, &found) < 0) {
goto error;
}
@@ -851,7 +863,9 @@ virDomainPCIAddressGetNextAddr(virDomainPCIAddressSetPtr addrs,
a.slot = bus->minSlot;
- if (virDomainPCIAddressFindUnusedFunctionOnBus(bus, &a, function,
+ if (virDomainPCIAddressFindUnusedFunctionOnBus(bus, &a,
+ aggregateSlotIdx,
+ function,
flags, &found) < 0) {
goto error;
}
@@ -910,12 +924,13 @@ virDomainPCIAddressReserveNextAddr(virDomainPCIAddressSetPtr addrs,
{
virPCIDeviceAddress addr;
- if (virDomainPCIAddressGetNextAddr(addrs, &addr, flags,
- dev->isolationGroup, function) < 0)
+ if (virDomainPCIAddressGetNextAddr(addrs, &addr, flags, dev->isolationGroup,
+ dev->aggregateSlotIdx, function) < 0)
return -1;
if (virDomainPCIAddressReserveAddrInternal(addrs, &addr, flags,
- dev->isolationGroup, false) < 0)
+ dev->isolationGroup,
+ dev->aggregateSlotIdx, false) < 0)
return -1;
if (!addrs->dryRun) {
diff --git a/src/conf/domain_addr.h b/src/conf/domain_addr.h
index d3541bab09..fa98b67e5c 100644
--- a/src/conf/domain_addr.h
+++ b/src/conf/domain_addr.h
@@ -32,23 +32,18 @@
typedef enum {
VIR_PCI_CONNECT_HOTPLUGGABLE = 1 << 0, /* is hotplug needed/supported */
- /* set for devices that can share a single slot in auto-assignment
- * (by assigning one device to each of the 8 functions on the slot)
- */
- VIR_PCI_CONNECT_AGGREGATE_SLOT = 1 << 1,
-
/* kinds of devices as a bitmap so they can be combined (some PCI
* controllers permit connecting multiple types of devices)
*/
- VIR_PCI_CONNECT_TYPE_PCI_DEVICE = 1 << 2,
- VIR_PCI_CONNECT_TYPE_PCIE_DEVICE = 1 << 3,
- VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT = 1 << 4,
- VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_UPSTREAM_PORT = 1 << 5,
- VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_DOWNSTREAM_PORT = 1 << 6,
- VIR_PCI_CONNECT_TYPE_DMI_TO_PCI_BRIDGE = 1 << 7,
- VIR_PCI_CONNECT_TYPE_PCI_EXPANDER_BUS = 1 << 8,
- VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS = 1 << 9,
- VIR_PCI_CONNECT_TYPE_PCI_BRIDGE = 1 << 10,
+ VIR_PCI_CONNECT_TYPE_PCI_DEVICE = 1 << 1,
+ VIR_PCI_CONNECT_TYPE_PCIE_DEVICE = 1 << 2,
+ VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT = 1 << 3,
+ VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_UPSTREAM_PORT = 1 << 4,
+ VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_DOWNSTREAM_PORT = 1 << 5,
+ VIR_PCI_CONNECT_TYPE_DMI_TO_PCI_BRIDGE = 1 << 6,
+ VIR_PCI_CONNECT_TYPE_PCI_EXPANDER_BUS = 1 << 7,
+ VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS = 1 << 8,
+ VIR_PCI_CONNECT_TYPE_PCI_BRIDGE = 1 << 9,
} virDomainPCIConnectFlags;
/* a combination of all bits that describe the type of connections
@@ -81,12 +76,12 @@ typedef struct {
*/
uint8_t functions;
- /* aggregate is true if this slot has only devices with
- * VIR_PCI_CONNECT_AGGREGATE assigned to its functions (meaning
- * that other devices with the same flags could also be
- * auto-assigned to the other functions)
+ /* aggregate is greater than zero if this slot has only devices with
+ * VIR_PCI_CONNECT_AGGREGATE assigned to its functions and
+ * that other devices with the same aggregateSlotIdx could also be
+ * auto-assigned to the other functions on this slot)
*/
- bool aggregate;
+ unsigned int aggregateSlotIdx;
} virDomainPCIAddressSlot;
typedef struct {
@@ -152,7 +147,8 @@ bool virDomainPCIAddressSlotInUse(virDomainPCIAddressSetPtr addrs,
int virDomainPCIAddressReserveAddr(virDomainPCIAddressSetPtr addrs,
virPCIDeviceAddressPtr addr,
virDomainPCIConnectFlags flags,
- unsigned int isolationGroup)
+ unsigned int isolationGroup,
+ unsigned int aggregateSlotIdx)
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
int virDomainPCIAddressReserveNextAddr(virDomainPCIAddressSetPtr addrs,
diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c
index 5f4e8edd2c..424b56dac9 100644
--- a/src/qemu/qemu_domain_address.c
+++ b/src/qemu/qemu_domain_address.c
@@ -26,6 +26,7 @@
#include "qemu_domain_address.h"
#include "qemu_domain.h"
#include "viralloc.h"
+#include "virhostdev.h"
#include "virerror.h"
#include "virlog.h"
@@ -1185,6 +1186,53 @@ qemuDomainSetupIsolationGroups(virDomainDefPtr def)
}
+void
+qemuDomainSetDeviceSlotAggregateIdx(virDomainDefPtr def ATTRIBUTE_UNUSED,
+ virDomainDeviceDefPtr dev)
+{
+ virDomainDeviceInfoPtr info = virDomainDeviceGetInfo(dev);
+
+ if (!info)
+ return;
+
+ info->aggregateSlotIdx = 0;
+
+ if (dev->type == VIR_DOMAIN_DEVICE_CONTROLLER) {
+ virDomainControllerDefPtr cont = dev->data.controller;
+ if (cont->type == VIR_DOMAIN_CONTROLLER_TYPE_PCI &&
+ cont->model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT_PORT) {
+ info->aggregateSlotIdx = 1;
+ }
+ }
+
+ return;
+}
+
+
+static int
+qemuDomainFillDeviceSlotAggregationIter(virDomainDefPtr def,
+ virDomainDeviceDefPtr dev,
+ virDomainDeviceInfoPtr info ATTRIBUTE_UNUSED,
+ void *opaque ATTRIBUTE_UNUSED)
+{
+ qemuDomainSetDeviceSlotAggregateIdx(def, dev);
+
+ return 0;
+}
+
+
+static int
+qemuDomainSetupSlotAggregation(virDomainDefPtr def)
+{
+ if (virDomainDeviceInfoIterate(def, qemuDomainFillDeviceSlotAggregationIter,
+ NULL) < 0) {
+ return -1;
+ }
+
+ return 0;
+}
+
+
/**
* qemuDomainFillDevicePCIConnectFlags:
*
@@ -1319,7 +1367,8 @@ qemuDomainCollectPCIAddress(virDomainDefPtr def ATTRIBUTE_UNUSED,
if (virDomainPCIAddressReserveAddr(addrs, addr,
info->pciConnectFlags,
- info->isolationGroup) < 0) {
+ info->isolationGroup,
+ info->aggregateSlotIdx) < 0) {
goto cleanup;
}
@@ -1480,7 +1529,7 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def,
continue;
}
if (addrs->nbuses &&
- virDomainPCIAddressReserveAddr(addrs, &cont->info.addr.pci, flags, 0) < 0)
+ virDomainPCIAddressReserveAddr(addrs, &cont->info.addr.pci, flags, 0, 0) < 0)
goto cleanup;
}
@@ -1489,11 +1538,11 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def,
memset(&tmp_addr, 0, sizeof(tmp_addr));
tmp_addr.slot = 1;
/* ISA Bridge at 00:01.0 */
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
goto cleanup;
/* Bridge at 00:01.3 */
tmp_addr.function = 3;
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
goto cleanup;
}
@@ -1528,7 +1577,7 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def,
goto cleanup;
}
} else {
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
goto cleanup;
primaryVideo->info.addr.pci = tmp_addr;
primaryVideo->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
@@ -1553,7 +1602,7 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def,
VIR_DEBUG("PCI address 0:0:2.0 in use, future addition of a video"
" device will not be possible without manual"
" intervention");
- } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) {
+ } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0) {
goto cleanup;
}
}
@@ -1629,7 +1678,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
assign = true;
}
if (assign) {
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
goto cleanup;
cont->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
@@ -1652,7 +1701,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
memset(&tmp_addr, 0, sizeof(tmp_addr));
tmp_addr.slot = 0x1E;
if (!virDomainPCIAddressSlotInUse(addrs, &tmp_addr)) {
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
goto cleanup;
cont->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
@@ -1676,12 +1725,12 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
tmp_addr.slot = 0x1F;
tmp_addr.function = 0;
tmp_addr.multi = VIR_TRISTATE_SWITCH_ON;
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
goto cleanup;
tmp_addr.function = 3;
tmp_addr.multi = VIR_TRISTATE_SWITCH_ABSENT;
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
goto cleanup;
}
@@ -1715,7 +1764,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
goto cleanup;
}
} else {
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
goto cleanup;
primaryVideo->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
primaryVideo->info.addr.pci = tmp_addr;
@@ -1741,7 +1790,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
" device will not be possible without manual"
" intervention");
virResetLastError();
- } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) {
+ } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0) {
goto cleanup;
}
}
@@ -1762,7 +1811,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
!virDeviceInfoPCIAddressWanted(&sound->info)) {
continue;
}
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
goto cleanup;
sound->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
@@ -1967,7 +2016,8 @@ qemuDomainAssignDevicePCISlots(virDomainDefPtr def,
/* Reserve this function on the slot we found */
if (virDomainPCIAddressReserveAddr(addrs, &addr,
cont->info.pciConnectFlags,
- cont->info.isolationGroup) < 0) {
+ cont->info.isolationGroup,
+ cont->info.aggregateSlotIdx) < 0) {
goto error;
}
@@ -2348,6 +2398,9 @@ qemuDomainAssignPCIAddresses(virDomainDefPtr def,
if (qemuDomainSetupIsolationGroups(def) < 0)
goto cleanup;
+ if (qemuDomainSetupSlotAggregation(def) < 0)
+ goto cleanup;
+
if (nbuses > 0) {
/* 1st pass to figure out how many PCI bridges we need */
if (!(addrs = qemuDomainPCIAddressSetCreate(def, nbuses, true)))
@@ -2463,6 +2516,7 @@ qemuDomainAssignPCIAddresses(virDomainDefPtr def,
dev.data.controller = def->controllers[contIndex];
/* set connect flags so it will be properly addressed */
qemuDomainFillDevicePCIConnectFlags(def, &dev, qemuCaps, driver);
+ qemuDomainSetDeviceSlotAggregateIdx(def, &dev);
/* Reserve an address for the controller. pci-root and pcie-root
* controllers don't plug into any other PCI controller, hence
@@ -2932,6 +2986,7 @@ qemuDomainEnsurePCIAddress(virDomainObjPtr obj,
return 0;
qemuDomainFillDevicePCIConnectFlags(obj->def, dev, priv->qemuCaps, driver);
+ qemuDomainSetDeviceSlotAggregateIdx(obj->def, dev);
return virDomainPCIAddressEnsureAddr(priv->pciaddrs, info,
info->pciConnectFlags);
diff --git a/src/qemu/qemu_domain_address.h b/src/qemu/qemu_domain_address.h
index 83f8e81cad..650f977cf7 100644
--- a/src/qemu/qemu_domain_address.h
+++ b/src/qemu/qemu_domain_address.h
@@ -55,6 +55,14 @@ int qemuDomainFillDeviceIsolationGroup(virDomainDefPtr def,
virDomainDeviceDefPtr dev)
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
+void
+qemuDomainSetDeviceSlotAggregateIdx(virDomainDefPtr def,
+ virDomainDeviceDefPtr dev);
+int
+qemuDomainDefDeviceFindSlotAggregateIdx(virDomainDefPtr def,
+ virDomainDeviceDefPtr dev);
+
+
void qemuDomainReleaseDeviceAddress(virDomainObjPtr vm,
virDomainDeviceInfoPtr info,
const char *devstr);
More information about the libvir-list
mailing list