/*
* Pre-condition: driver->inactivePciHostdevs & driver->activePciHostdevs
* are locked
*/
void qemuReattachPciDevice(virPCIDevicePtr dev, virQEMUDriverPtr driver)
{
int retries = 100;
/* If the device is not managed and was attached to guest
* successfully, it must have been inactive.
*/
if (!virPCIDeviceGetManaged(dev)) {
if (virPCIDeviceListAdd(driver->inactivePciHostdevs, dev) < 0)
virPCIDeviceFree(dev);
return;
}
while (virPCIDeviceWaitForCleanup(dev, "kvm_assigned_device")
&& retries) {
usleep(100*1000);
retries--;
}
if (virPCIDeviceReattach(dev, driver->activePciHostdevs,
driver->inactivePciHostdevs) < 0) {
virErrorPtr err = virGetLastError();
VIR_ERROR(_("Failed to re-attach PCI device: %s"),
err ? err->message : _("unknown error"));
virResetError(err);
}
virPCIDeviceFree(dev);
}
int
qemuUpdateActivePciHostdevs(virQEMUDriverPtr driver,
virDomainDefPtr def)
{
virDomainHostdevDefPtr hostdev = NULL;
virPCIDevicePtr dev = NULL;
size_t i;
int ret = -1;
if (!def->nhostdevs)
return 0;
virObjectLock(driver->activePciHostdevs);
virObjectLock(driver->inactivePciHostdevs);
for (i = 0; i < def->nhostdevs; i++) {
hostdev = def->hostdevs[i];
if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
continue;
if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
continue;
dev = virPCIDeviceNew(hostdev->source.subsys.u.pci.addr.domain,
hostdev->source.subsys.u.pci.addr.bus,
hostdev->source.subsys.u.pci.addr.slot,
hostdev->source.subsys.u.pci.addr.function);
if (!dev)
goto cleanup;
virPCIDeviceSetManaged(dev, hostdev->managed);
if (hostdev->source.subsys.u.pci.backend
== VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO) {
if (virPCIDeviceSetStubDriver(dev, "vfio-pci") < 0)
goto cleanup;
} else {
if (virPCIDeviceSetStubDriver(dev, "pci-stub") < 0)
goto cleanup;
}
virPCIDeviceSetUsedBy(dev, def->name);
/* Setup the original states for the PCI device */
virPCIDeviceSetUnbindFromStub(dev, hostdev->origstates.states.pci.unbind_from_stub);
virPCIDeviceSetRemoveSlot(dev, hostdev->origstates.states.pci.remove_slot);
virPCIDeviceSetReprobe(dev, hostdev->origstates.states.pci.reprobe);
if (virPCIDeviceListAdd(driver->activePciHostdevs, dev) < 0)
goto cleanup;
dev = NULL;
}
ret = 0;
cleanup:
virPCIDeviceFree(dev);
virObjectUnlock(driver->activePciHostdevs);
virObjectUnlock(driver->inactivePciHostdevs);
return ret;
}
int
qemuTeardownHostdevCgroup(virDomainObjPtr vm,
virDomainHostdevDefPtr dev)
{
int ret = -1;
qemuDomainObjPrivatePtr priv = vm->privateData;
virDomainHostdevSubsysPCIPtr pcisrc = &dev->source.subsys.u.pci;
virPCIDevicePtr pci = NULL;
char *path = NULL;
/* currently this only does something for PCI devices using vfio
* for device assignment, but it is called for *all* hostdev
* devices.
*/
if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_DEVICES))
return 0;
if (dev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) {
switch (dev->source.subsys.type) {
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
if (pcisrc->backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO) {
int rv;
pci = virPCIDeviceNew(pcisrc->addr.domain,
pcisrc->addr.bus,
pcisrc->addr.slot,
pcisrc->addr.function);
if (!pci)
goto cleanup;
if (!(path = virPCIDeviceGetIOMMUGroupDev(pci)))
goto cleanup;
VIR_DEBUG("Cgroup deny %s for PCI device assignment", path);
rv = virCgroupDenyDevicePath(priv->cgroup, path,
VIR_CGROUP_DEVICE_RWM);
virDomainAuditCgroupPath(vm, priv->cgroup,
"deny", path, "rwm", rv == 0);
if (rv < 0)
goto cleanup;
}
break;
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
/* nothing to tear down for USB */
break;
default:
break;
}
}
ret = 0;
cleanup:
virPCIDeviceFree(pci);
VIR_FREE(path);
return ret;
}
/*
* Pre-condition: driver->activePciHostdevs is locked
*/
static virPCIDeviceListPtr
qemuGetActivePciHostDeviceList(virQEMUDriverPtr driver,
virDomainHostdevDefPtr *hostdevs,
int nhostdevs)
{
virPCIDeviceListPtr list;
int i;
if (!(list = virPCIDeviceListNew()))
return NULL;
for (i = 0 ; i < nhostdevs ; i++) {
virDomainHostdevDefPtr hostdev = hostdevs[i];
virPCIDevicePtr dev;
virPCIDevicePtr activeDev;
if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
continue;
if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
continue;
dev = virPCIDeviceNew(hostdev->source.subsys.u.pci.addr.domain,
hostdev->source.subsys.u.pci.addr.bus,
hostdev->source.subsys.u.pci.addr.slot,
hostdev->source.subsys.u.pci.addr.function);
if (!dev) {
virObjectUnref(list);
return NULL;
}
if ((activeDev = virPCIDeviceListFind(driver->activePciHostdevs, dev))) {
if (virPCIDeviceListAdd(list, activeDev) < 0) {
virPCIDeviceFree(dev);
virObjectUnref(list);
return NULL;
}
}
virPCIDeviceFree(dev);
}
return list;
}
static virPCIDeviceListPtr
qemuGetPciHostDeviceList(virDomainHostdevDefPtr *hostdevs, int nhostdevs)
{
virPCIDeviceListPtr list;
size_t i;
if (!(list = virPCIDeviceListNew()))
return NULL;
for (i = 0; i < nhostdevs; i++) {
virDomainHostdevDefPtr hostdev = hostdevs[i];
virPCIDevicePtr dev;
if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
continue;
if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
continue;
dev = virPCIDeviceNew(hostdev->source.subsys.u.pci.addr.domain,
hostdev->source.subsys.u.pci.addr.bus,
hostdev->source.subsys.u.pci.addr.slot,
hostdev->source.subsys.u.pci.addr.function);
if (!dev) {
virObjectUnref(list);
return NULL;
}
if (virPCIDeviceListAdd(list, dev) < 0) {
virPCIDeviceFree(dev);
virObjectUnref(list);
return NULL;
}
virPCIDeviceSetManaged(dev, hostdev->managed);
if (hostdev->source.subsys.u.pci.backend
== VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO) {
if (virPCIDeviceSetStubDriver(dev, "vfio-pci") < 0) {
virObjectUnref(list);
return NULL;
}
} else {
if (virPCIDeviceSetStubDriver(dev, "pci-stub") < 0) {
virObjectUnref(list);
return NULL;
}
}
}
return list;
}
static void
myCleanup(void)
{
size_t i;
for (i = 0; i < nhostdevs; i++) {
virPCIDeviceFree(dev[i]);
virDomainHostdevDefFree(hostdevs[i]);
}
if (mgr) {
virObjectUnref(mgr->activePCIHostdevs);
virObjectUnref(mgr->inactivePCIHostdevs);
virObjectUnref(mgr->activeUSBHostdevs);
VIR_FREE(mgr->stateDir);
VIR_FREE(mgr);
}
}
static void
myCleanup(void)
{
size_t i;
for (i = 0; i < nhostdevs; i++) {
virPCIDeviceFree(dev[i]);
virDomainHostdevDefFree(hostdevs[i]);
}
if (mgr) {
if (!getenv("LIBVIRT_SKIP_CLEANUP"))
virFileDeleteTree(mgr->stateDir);
virObjectUnref(mgr->activePCIHostdevs);
virObjectUnref(mgr->inactivePCIHostdevs);
virObjectUnref(mgr->activeUSBHostdevs);
VIR_FREE(mgr->stateDir);
VIR_FREE(mgr);
}
}
int qemuPrepareHostdevPCIDevices(virQEMUDriverPtr driver,
const char *name,
const unsigned char *uuid,
virDomainHostdevDefPtr *hostdevs,
int nhostdevs)
{
virPCIDeviceListPtr pcidevs;
int last_processed_hostdev_vf = -1;
int i;
int ret = -1;
virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
virObjectLock(driver->activePciHostdevs);
virObjectLock(driver->inactivePciHostdevs);
if (!(pcidevs = qemuGetPciHostDeviceList(hostdevs, nhostdevs)))
goto cleanup;
/* We have to use 9 loops here. *All* devices must
* be detached before we reset any of them, because
* in some cases you have to reset the whole PCI,
* which impacts all devices on it. Also, all devices
* must be reset before being marked as active.
*/
/* Loop 1: validate that non-managed device isn't in use, eg
* by checking that device is either un-bound, or bound
* to pci-stub.ko
*/
for (i = 0; i < virPCIDeviceListCount(pcidevs); i++) {
virPCIDevicePtr dev = virPCIDeviceListGet(pcidevs, i);
virPCIDevicePtr other;
if (!virPCIDeviceIsAssignable(dev, !cfg->relaxedACS)) {
virReportError(VIR_ERR_OPERATION_INVALID,
_("PCI device %s is not assignable"),
virPCIDeviceGetName(dev));
goto cleanup;
}
/* The device is in use by other active domain if
* the dev is in list driver->activePciHostdevs.
*/
if ((other = virPCIDeviceListFind(driver->activePciHostdevs, dev))) {
const char *other_name = virPCIDeviceGetUsedBy(other);
if (other_name)
virReportError(VIR_ERR_OPERATION_INVALID,
_("PCI device %s is in use by domain %s"),
virPCIDeviceGetName(dev), other_name);
else
virReportError(VIR_ERR_OPERATION_INVALID,
_("PCI device %s is already in use"),
virPCIDeviceGetName(dev));
goto cleanup;
}
}
/* Loop 2: detach managed devices (i.e. bind to appropriate stub driver) */
for (i = 0; i < virPCIDeviceListCount(pcidevs); i++) {
virPCIDevicePtr dev = virPCIDeviceListGet(pcidevs, i);
if (virPCIDeviceGetManaged(dev) &&
virPCIDeviceDetach(dev, driver->activePciHostdevs, NULL, NULL) < 0)
goto reattachdevs;
}
/* Loop 3: Now that all the PCI hostdevs have been detached, we
* can safely reset them */
for (i = 0; i < virPCIDeviceListCount(pcidevs); i++) {
virPCIDevicePtr dev = virPCIDeviceListGet(pcidevs, i);
if (virPCIDeviceReset(dev, driver->activePciHostdevs,
driver->inactivePciHostdevs) < 0)
goto reattachdevs;
}
/* Loop 4: For SRIOV network devices, Now that we have detached the
* the network device, set the netdev config */
for (i = 0; i < nhostdevs; i++) {
virDomainHostdevDefPtr hostdev = hostdevs[i];
if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
continue;
if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
continue;
if (hostdev->parent.type == VIR_DOMAIN_DEVICE_NET &&
hostdev->parent.data.net) {
if (qemuDomainHostdevNetConfigReplace(hostdev, uuid,
cfg->stateDir) < 0) {
goto resetvfnetconfig;
}
}
last_processed_hostdev_vf = i;
}
/* Loop 5: Now mark all the devices as active */
for (i = 0; i < virPCIDeviceListCount(pcidevs); i++) {
virPCIDevicePtr dev = virPCIDeviceListGet(pcidevs, i);
if (virPCIDeviceListAdd(driver->activePciHostdevs, dev) < 0)
goto inactivedevs;
}
/* Loop 6: Now remove the devices from inactive list. */
for (i = 0; i < virPCIDeviceListCount(pcidevs); i++) {
virPCIDevicePtr dev = virPCIDeviceListGet(pcidevs, i);
virPCIDeviceListDel(driver->inactivePciHostdevs, dev);
}
/* Loop 7: Now set the used_by_domain of the device in
* driver->activePciHostdevs as domain name.
*/
for (i = 0; i < virPCIDeviceListCount(pcidevs); i++) {
virPCIDevicePtr dev, activeDev;
dev = virPCIDeviceListGet(pcidevs, i);
activeDev = virPCIDeviceListFind(driver->activePciHostdevs, dev);
if (activeDev)
virPCIDeviceSetUsedBy(activeDev, name);
}
/* Loop 8: Now set the original states for hostdev def */
for (i = 0; i < nhostdevs; i++) {
virPCIDevicePtr dev;
virPCIDevicePtr pcidev;
virDomainHostdevDefPtr hostdev = hostdevs[i];
if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
continue;
if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
continue;
dev = virPCIDeviceNew(hostdev->source.subsys.u.pci.addr.domain,
hostdev->source.subsys.u.pci.addr.bus,
hostdev->source.subsys.u.pci.addr.slot,
hostdev->source.subsys.u.pci.addr.function);
/* original states "unbind_from_stub", "remove_slot",
* "reprobe" were already set by pciDettachDevice in
* loop 2.
*/
if ((pcidev = virPCIDeviceListFind(pcidevs, dev))) {
hostdev->origstates.states.pci.unbind_from_stub =
virPCIDeviceGetUnbindFromStub(pcidev);
hostdev->origstates.states.pci.remove_slot =
virPCIDeviceGetRemoveSlot(pcidev);
hostdev->origstates.states.pci.reprobe =
virPCIDeviceGetReprobe(pcidev);
}
virPCIDeviceFree(dev);
}
/* Loop 9: Now steal all the devices from pcidevs */
while (virPCIDeviceListCount(pcidevs) > 0)
virPCIDeviceListStealIndex(pcidevs, 0);
ret = 0;
goto cleanup;
inactivedevs:
/* Only steal all the devices from driver->activePciHostdevs. We will
* free them in virObjectUnref().
*/
while (virPCIDeviceListCount(pcidevs) > 0) {
virPCIDevicePtr dev = virPCIDeviceListGet(pcidevs, 0);
virPCIDeviceListSteal(driver->activePciHostdevs, dev);
}
resetvfnetconfig:
for (i = 0; i < last_processed_hostdev_vf; i++) {
virDomainHostdevDefPtr hostdev = hostdevs[i];
if (hostdev->parent.type == VIR_DOMAIN_DEVICE_NET &&
hostdev->parent.data.net) {
qemuDomainHostdevNetConfigRestore(hostdev, cfg->stateDir);
}
}
reattachdevs:
for (i = 0; i < virPCIDeviceListCount(pcidevs); i++) {
virPCIDevicePtr dev = virPCIDeviceListGet(pcidevs, i);
/* NB: This doesn't actually re-bind to original driver, just
* unbinds from the stub driver
*/
virPCIDeviceReattach(dev, driver->activePciHostdevs, NULL);
}
cleanup:
virObjectUnlock(driver->activePciHostdevs);
virObjectUnlock(driver->inactivePciHostdevs);
virObjectUnref(pcidevs);
virObjectUnref(cfg);
return ret;
}
int
qemuSetupHostdevCGroup(virDomainObjPtr vm,
virDomainHostdevDefPtr dev)
{
int ret = -1;
qemuDomainObjPrivatePtr priv = vm->privateData;
virDomainHostdevSubsysUSBPtr usbsrc = &dev->source.subsys.u.usb;
virDomainHostdevSubsysPCIPtr pcisrc = &dev->source.subsys.u.pci;
virDomainHostdevSubsysSCSIPtr scsisrc = &dev->source.subsys.u.scsi;
virPCIDevicePtr pci = NULL;
virUSBDevicePtr usb = NULL;
virSCSIDevicePtr scsi = NULL;
char *path = NULL;
/* currently this only does something for PCI devices using vfio
* for device assignment, but it is called for *all* hostdev
* devices.
*/
if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_DEVICES))
return 0;
if (dev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) {
switch (dev->source.subsys.type) {
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
if (pcisrc->backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO) {
int rv;
pci = virPCIDeviceNew(pcisrc->addr.domain,
pcisrc->addr.bus,
pcisrc->addr.slot,
pcisrc->addr.function);
if (!pci)
goto cleanup;
if (!(path = virPCIDeviceGetIOMMUGroupDev(pci)))
goto cleanup;
VIR_DEBUG("Cgroup allow %s for PCI device assignment", path);
rv = virCgroupAllowDevicePath(priv->cgroup, path,
VIR_CGROUP_DEVICE_RW);
virDomainAuditCgroupPath(vm, priv->cgroup,
"allow", path, "rw", rv == 0);
if (rv < 0)
goto cleanup;
}
break;
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
/* NB: hostdev->missing wasn't previously checked in the
* case of hotplug, only when starting a domain. Now it is
* always checked, and the cgroup setup skipped if true.
*/
if (dev->missing)
break;
if ((usb = virUSBDeviceNew(usbsrc->bus, usbsrc->device,
NULL)) == NULL) {
goto cleanup;
}
/* oddly, qemuSetupHostUSBDeviceCgroup doesn't ever
* reference the usb object we just created
*/
if (virUSBDeviceFileIterate(usb, qemuSetupHostUSBDeviceCgroup,
vm) < 0) {
goto cleanup;
}
break;
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI: {
if (scsisrc->protocol ==
VIR_DOMAIN_HOSTDEV_SCSI_PROTOCOL_TYPE_ISCSI) {
virDomainHostdevSubsysSCSIiSCSIPtr iscsisrc = &scsisrc->u.iscsi;
/* Follow qemuSetupDiskCgroup() and qemuSetImageCgroupInternal()
* which does nothing for non local storage
*/
VIR_DEBUG("Not updating cgroups for hostdev iSCSI path '%s'",
iscsisrc->path);
} else {
virDomainHostdevSubsysSCSIHostPtr scsihostsrc =
&scsisrc->u.host;
if ((scsi = virSCSIDeviceNew(NULL,
scsihostsrc->adapter,
scsihostsrc->bus,
scsihostsrc->target,
scsihostsrc->unit,
dev->readonly,
dev->shareable)) == NULL)
goto cleanup;
if (virSCSIDeviceFileIterate(scsi,
qemuSetupHostSCSIDeviceCgroup,
vm) < 0)
goto cleanup;
}
break;
}
default:
break;
}
}
ret = 0;
cleanup:
virPCIDeviceFree(pci);
virUSBDeviceFree(usb);
virSCSIDeviceFree(scsi);
VIR_FREE(path);
return ret;
}
int
qemuSetupHostdevCGroup(virDomainObjPtr vm,
virDomainHostdevDefPtr dev)
{
int ret = -1;
qemuDomainObjPrivatePtr priv = vm->privateData;
virPCIDevicePtr pci = NULL;
virUSBDevicePtr usb = NULL;
virSCSIDevicePtr scsi = NULL;
char *path = NULL;
/* currently this only does something for PCI devices using vfio
* for device assignment, but it is called for *all* hostdev
* devices.
*/
if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_DEVICES))
return 0;
if (dev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) {
switch (dev->source.subsys.type) {
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
if (dev->source.subsys.u.pci.backend
== VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO) {
int rv;
pci = virPCIDeviceNew(dev->source.subsys.u.pci.addr.domain,
dev->source.subsys.u.pci.addr.bus,
dev->source.subsys.u.pci.addr.slot,
dev->source.subsys.u.pci.addr.function);
if (!pci)
goto cleanup;
if (!(path = virPCIDeviceGetIOMMUGroupDev(pci)))
goto cleanup;
VIR_DEBUG("Cgroup allow %s for PCI device assignment", path);
rv = virCgroupAllowDevicePath(priv->cgroup, path,
VIR_CGROUP_DEVICE_RW);
virDomainAuditCgroupPath(vm, priv->cgroup,
"allow", path, "rw", rv == 0);
if (rv < 0)
goto cleanup;
}
break;
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
/* NB: hostdev->missing wasn't previously checked in the
* case of hotplug, only when starting a domain. Now it is
* always checked, and the cgroup setup skipped if true.
*/
if (dev->missing)
break;
if ((usb = virUSBDeviceNew(dev->source.subsys.u.usb.bus,
dev->source.subsys.u.usb.device,
NULL)) == NULL) {
goto cleanup;
}
/* oddly, qemuSetupHostUSBDeviceCgroup doesn't ever
* reference the usb object we just created
*/
if (virUSBDeviceFileIterate(usb, qemuSetupHostUSBDeviceCgroup,
vm) < 0) {
goto cleanup;
}
break;
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI:
if ((scsi = virSCSIDeviceNew(NULL,
dev->source.subsys.u.scsi.adapter,
dev->source.subsys.u.scsi.bus,
dev->source.subsys.u.scsi.target,
dev->source.subsys.u.scsi.unit,
dev->readonly,
dev->shareable)) == NULL)
goto cleanup;
if (virSCSIDeviceFileIterate(scsi,
qemuSetupHostSCSIDeviceCgroup,
vm) < 0)
goto cleanup;
default:
break;
}
}
ret = 0;
cleanup:
virPCIDeviceFree(pci);
virUSBDeviceFree(usb);
virSCSIDeviceFree(scsi);
VIR_FREE(path);
return ret;
}