int
qemuSetupCpusetMems(virDomainObjPtr vm)
{
virCgroupPtr cgroup_temp = NULL;
qemuDomainObjPrivatePtr priv = vm->privateData;
virDomainNumatuneMemMode mode;
char *mem_mask = NULL;
int ret = -1;
if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
return 0;
if (virDomainNumatuneGetMode(vm->def->numa, -1, &mode) < 0 ||
mode != VIR_DOMAIN_NUMATUNE_MEM_STRICT)
return 0;
if (virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
priv->autoNodeset,
&mem_mask, -1) < 0)
goto cleanup;
if (mem_mask)
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,
false, &cgroup_temp) < 0 ||
virCgroupSetCpusetMems(cgroup_temp, mem_mask) < 0)
goto cleanup;
ret = 0;
cleanup:
VIR_FREE(mem_mask);
virCgroupFree(&cgroup_temp);
return ret;
}
int
qemuSetupCgroupForExtDevices(virDomainObjPtr vm,
virQEMUDriverPtr driver)
{
qemuDomainObjPrivatePtr priv = vm->privateData;
virCgroupPtr cgroup_temp = NULL;
int ret = -1;
if (!qemuExtDevicesHasDevice(vm->def) ||
priv->cgroup == NULL)
return 0; /* Not supported, so claim success */
/*
* If CPU cgroup controller is not initialized here, then we need
* neither period nor quota settings. And if CPUSET controller is
* not initialized either, then there's nothing to do anyway.
*/
if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
return 0;
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,
false, &cgroup_temp) < 0)
goto cleanup;
ret = qemuExtDevicesSetupCgroup(driver, vm->def, cgroup_temp);
cleanup:
virCgroupFree(&cgroup_temp);
return ret;
}
/**
* qemuCgroupEmulatorAllNodesAllow:
* @cgroup: domain cgroup pointer
* @retData: filled with structure used to roll back the operation
*
* Allows all NUMA nodes for the qemu emulator thread temporarily. This is
* necessary when hotplugging cpus since it requires memory allocated in the
* DMA region. Afterwards the operation can be reverted by
* qemuCgroupEmulatorAllNodesRestore.
*
* Returns 0 on success -1 on error
*/
int
qemuCgroupEmulatorAllNodesAllow(virCgroupPtr cgroup,
qemuCgroupEmulatorAllNodesDataPtr *retData)
{
qemuCgroupEmulatorAllNodesDataPtr data = NULL;
char *all_nodes_str = NULL;
virBitmapPtr all_nodes = NULL;
int ret = -1;
if (!virNumaIsAvailable() ||
!virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
return 0;
if (!(all_nodes = virNumaGetHostMemoryNodeset()))
goto cleanup;
if (!(all_nodes_str = virBitmapFormat(all_nodes)))
goto cleanup;
if (VIR_ALLOC(data) < 0)
goto cleanup;
if (virCgroupNewThread(cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,
false, &data->emulatorCgroup) < 0)
goto cleanup;
if (virCgroupGetCpusetMems(data->emulatorCgroup, &data->emulatorMemMask) < 0 ||
virCgroupSetCpusetMems(data->emulatorCgroup, all_nodes_str) < 0)
goto cleanup;
VIR_STEAL_PTR(*retData, data);
ret = 0;
cleanup:
VIR_FREE(all_nodes_str);
virBitmapFree(all_nodes);
qemuCgroupEmulatorAllNodesDataFree(data);
return ret;
}
int
qemuSetupCgroupForVcpu(virDomainObjPtr vm)
{
virCgroupPtr cgroup_vcpu = NULL;
qemuDomainObjPrivatePtr priv = vm->privateData;
virDomainDefPtr def = vm->def;
size_t i, j;
unsigned long long period = vm->def->cputune.period;
long long quota = vm->def->cputune.quota;
char *mem_mask = NULL;
virDomainNumatuneMemMode mem_mode;
if ((period || quota) &&
!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("cgroup cpu is required for scheduler tuning"));
return -1;
}
/*
* If CPU cgroup controller is not initialized here, then we need
* neither period nor quota settings. And if CPUSET controller is
* not initialized either, then there's nothing to do anyway.
*/
if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
return 0;
/* We are trying to setup cgroups for CPU pinning, which can also be done
* with virProcessSetAffinity, thus the lack of cgroups is not fatal here.
*/
if (priv->cgroup == NULL)
return 0;
if (priv->nvcpupids == 0 || priv->vcpupids[0] == vm->pid) {
/* If we don't know VCPU<->PID mapping or all vcpu runs in the same
* thread, we cannot control each vcpu.
*/
return 0;
}
if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 &&
mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
priv->autoNodeset,
&mem_mask, -1) < 0)
goto cleanup;
for (i = 0; i < priv->nvcpupids; i++) {
virCgroupFree(&cgroup_vcpu);
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, i,
true, &cgroup_vcpu) < 0)
goto cleanup;
/* move the thread for vcpu to sub dir */
if (virCgroupAddTask(cgroup_vcpu, priv->vcpupids[i]) < 0)
goto cleanup;
if (period || quota) {
if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0)
goto cleanup;
}
/* Set vcpupin in cgroup if vcpupin xml is provided */
if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
virBitmapPtr cpumap = NULL;
if (mem_mask &&
virCgroupSetCpusetMems(cgroup_vcpu, mem_mask) < 0)
goto cleanup;
/* try to use the default cpu maps */
if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
cpumap = priv->autoCpuset;
else
cpumap = vm->def->cpumask;
/* lookup a more specific pinning info */
for (j = 0; j < def->cputune.nvcpupin; j++) {
if (def->cputune.vcpupin[j]->id == i) {
cpumap = def->cputune.vcpupin[j]->cpumask;
break;
}
}
if (!cpumap)
continue;
if (qemuSetupCgroupCpusetCpus(cgroup_vcpu, cpumap) < 0)
goto cleanup;
}
}
virCgroupFree(&cgroup_vcpu);
VIR_FREE(mem_mask);
return 0;
cleanup:
if (cgroup_vcpu) {
virCgroupRemove(cgroup_vcpu);
virCgroupFree(&cgroup_vcpu);
}
VIR_FREE(mem_mask);
return -1;
}
static void
qemuRestoreCgroupState(virDomainObjPtr vm)
{
char *mem_mask = NULL;
char *nodeset = NULL;
int empty = -1;
qemuDomainObjPrivatePtr priv = vm->privateData;
size_t i = 0;
virBitmapPtr all_nodes;
virCgroupPtr cgroup_temp = NULL;
if (!(all_nodes = virNumaGetHostNodeset()))
goto error;
if (!(mem_mask = virBitmapFormat(all_nodes)))
goto error;
if ((empty = virCgroupHasEmptyTasks(priv->cgroup,
VIR_CGROUP_CONTROLLER_CPUSET)) <= 0)
goto error;
if (virCgroupSetCpusetMems(priv->cgroup, mem_mask) < 0)
goto error;
for (i = 0; i < priv->nvcpupids; i++) {
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, i,
false, &cgroup_temp) < 0 ||
virCgroupSetCpusetMemoryMigrate(cgroup_temp, true) < 0 ||
virCgroupGetCpusetMems(cgroup_temp, &nodeset) < 0 ||
virCgroupSetCpusetMems(cgroup_temp, nodeset) < 0)
goto cleanup;
VIR_FREE(nodeset);
virCgroupFree(&cgroup_temp);
}
for (i = 0; i < vm->def->niothreadids; i++) {
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD,
vm->def->iothreadids[i]->iothread_id,
false, &cgroup_temp) < 0 ||
virCgroupSetCpusetMemoryMigrate(cgroup_temp, true) < 0 ||
virCgroupGetCpusetMems(cgroup_temp, &nodeset) < 0 ||
virCgroupSetCpusetMems(cgroup_temp, nodeset) < 0)
goto cleanup;
VIR_FREE(nodeset);
virCgroupFree(&cgroup_temp);
}
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,
false, &cgroup_temp) < 0 ||
virCgroupSetCpusetMemoryMigrate(cgroup_temp, true) < 0 ||
virCgroupGetCpusetMems(cgroup_temp, &nodeset) < 0 ||
virCgroupSetCpusetMems(cgroup_temp, nodeset) < 0)
goto cleanup;
cleanup:
VIR_FREE(mem_mask);
VIR_FREE(nodeset);
virBitmapFree(all_nodes);
virCgroupFree(&cgroup_temp);
return;
error:
virResetLastError();
VIR_DEBUG("Couldn't restore cgroups to meaningful state");
goto cleanup;
}
int
qemuSetupCgroupForIOThreads(virDomainObjPtr vm)
{
virCgroupPtr cgroup_iothread = NULL;
qemuDomainObjPrivatePtr priv = vm->privateData;
virDomainDefPtr def = vm->def;
size_t i;
unsigned long long period = vm->def->cputune.period;
long long quota = vm->def->cputune.quota;
char *mem_mask = NULL;
virDomainNumatuneMemMode mem_mode;
if ((period || quota) &&
!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("cgroup cpu is required for scheduler tuning"));
return -1;
}
/*
* If CPU cgroup controller is not initialized here, then we need
* neither period nor quota settings. And if CPUSET controller is
* not initialized either, then there's nothing to do anyway.
*/
if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
return 0;
/* We are trying to setup cgroups for CPU pinning, which can also be done
* with virProcessSetAffinity, thus the lack of cgroups is not fatal here.
*/
if (priv->cgroup == NULL)
return 0;
if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 &&
mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
priv->autoNodeset,
&mem_mask, -1) < 0)
goto cleanup;
for (i = 0; i < def->niothreadids; i++) {
/* IOThreads are numbered 1..n, although the array is 0..n-1,
* so we will account for that here
*/
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD,
def->iothreadids[i]->iothread_id,
true, &cgroup_iothread) < 0)
goto cleanup;
/* move the thread for iothread to sub dir */
if (virCgroupAddTask(cgroup_iothread,
def->iothreadids[i]->thread_id) < 0)
goto cleanup;
if (period || quota) {
if (qemuSetupCgroupVcpuBW(cgroup_iothread, period, quota) < 0)
goto cleanup;
}
/* Set iothreadpin in cgroup if iothreadpin xml is provided */
if (virCgroupHasController(priv->cgroup,
VIR_CGROUP_CONTROLLER_CPUSET)) {
virBitmapPtr cpumask = NULL;
if (mem_mask &&
virCgroupSetCpusetMems(cgroup_iothread, mem_mask) < 0)
goto cleanup;
if (def->iothreadids[i]->cpumask)
cpumask = def->iothreadids[i]->cpumask;
else if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
cpumask = priv->autoCpuset;
else
cpumask = def->cpumask;
if (cpumask &&
qemuSetupCgroupCpusetCpus(cgroup_iothread, cpumask) < 0)
goto cleanup;
}
virCgroupFree(&cgroup_iothread);
}
VIR_FREE(mem_mask);
return 0;
cleanup:
if (cgroup_iothread) {
virCgroupRemove(cgroup_iothread);
virCgroupFree(&cgroup_iothread);
}
VIR_FREE(mem_mask);
return -1;
}
int
qemuSetupCgroupForEmulator(virDomainObjPtr vm)
{
virBitmapPtr cpumask = NULL;
virCgroupPtr cgroup_emulator = NULL;
virDomainDefPtr def = vm->def;
qemuDomainObjPrivatePtr priv = vm->privateData;
unsigned long long period = vm->def->cputune.emulator_period;
long long quota = vm->def->cputune.emulator_quota;
if ((period || quota) &&
!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("cgroup cpu is required for scheduler tuning"));
return -1;
}
/*
* If CPU cgroup controller is not initialized here, then we need
* neither period nor quota settings. And if CPUSET controller is
* not initialized either, then there's nothing to do anyway.
*/
if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
return 0;
if (priv->cgroup == NULL)
return 0; /* Not supported, so claim success */
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,
true, &cgroup_emulator) < 0)
goto cleanup;
if (virCgroupMoveTask(priv->cgroup, cgroup_emulator) < 0)
goto cleanup;
if (def->cputune.emulatorpin)
cpumask = def->cputune.emulatorpin;
else if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
cpumask = priv->autoCpuset;
else if (def->cpumask)
cpumask = def->cpumask;
if (cpumask) {
if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET) &&
qemuSetupCgroupCpusetCpus(cgroup_emulator, cpumask) < 0)
goto cleanup;
}
if (period || quota) {
if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
qemuSetupCgroupVcpuBW(cgroup_emulator, period,
quota) < 0)
goto cleanup;
}
virCgroupFree(&cgroup_emulator);
return 0;
cleanup:
if (cgroup_emulator) {
virCgroupRemove(cgroup_emulator);
virCgroupFree(&cgroup_emulator);
}
return -1;
}
