RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
{
if (idCpu >= MAXIMUM_PROCESSORS)
return false;
#if 0 /* this isn't safe at all IRQLs (great work guys) */
KAFFINITY Mask = KeQueryActiveProcessors();
return !!(Mask & RT_BIT_64(idCpu));
#else
return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
#endif
}
RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
{
#ifdef CONFIG_SMP
if (RT_UNLIKELY(idCpu >= NR_CPUS))
return false;
# ifdef cpu_online
return cpu_online(idCpu);
# else /* 2.4: */
return cpu_online_map & RT_BIT_64(idCpu);
# endif
#else
return idCpu == RTMpCpuId();
#endif
}
RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
{
#if 0
RTCPUID cCpus = rtMpDarwinMaxCpus();
return RTCpuSetFromU64(RT_BIT_64(cCpus) - 1);
#else
RTCpuSetEmpty(pSet);
RTCPUID cMax = rtMpDarwinMaxCpus();
for (RTCPUID idCpu = 0; idCpu < cMax; idCpu++)
if (RTMpIsCpuPossible(idCpu))
RTCpuSetAdd(pSet, idCpu);
return pSet;
#endif
}
/**
* Hook function for the thread-resumed event.
*
* @param pPreemptNotifier Pointer to the preempt_notifier struct.
* @param iCpu The CPU this thread is scheduled on.
*
* @remarks Called without holding the rq (runqueue) lock and with preemption
* enabled!
*/
static void rtThreadCtxHooksLnxSchedIn(struct preempt_notifier *pPreemptNotifier, int iCpu)
{
PRTTHREADCTXINT pThis = RT_FROM_MEMBER(pPreemptNotifier, RTTHREADCTXINT, hPreemptNotifier);
#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
RTCCUINTREG fSavedEFlags = ASMGetFlags();
stac();
#endif
AssertPtr(pThis);
AssertPtr(pThis->pfnThreadCtxHook);
Assert(pThis->fRegistered);
pThis->pfnThreadCtxHook(RTTHREADCTXEVENT_RESUMED, pThis->pvUser);
#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 19) && defined(RT_ARCH_AMD64)
fSavedEFlags &= ~RT_BIT_64(18) /*X86_EFL_AC*/;
fSavedEFlags |= pThis->fSavedRFlags & RT_BIT_64(18) /*X86_EFL_AC*/;
# endif
ASMSetFlags(fSavedEFlags);
#endif
}
/**
* Internal worker for the RTMpOn* APIs.
*
* @returns IPRT status code.
* @param pfnWorker The callback.
* @param pvUser1 User argument 1.
* @param pvUser2 User argument 2.
* @param enmCpuid What to do / is idCpu valid.
* @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
* RT_NT_CPUID_PAIR, otherwise ignored.
* @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
* @param pcHits Where to return the number of this. Optional.
*/
static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
{
PRTMPARGS pArgs;
KDPC *paExecCpuDpcs;
#if 0
/* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
* driver verifier doesn't complain...
*/
AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
#endif
#ifdef IPRT_TARGET_NT4
KAFFINITY Mask;
/* g_pfnrtNt* are not present on NT anyway. */
return VERR_NOT_SUPPORTED;
#else
KAFFINITY Mask = KeQueryActiveProcessors();
#endif
/* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
if (!g_pfnrtNtKeFlushQueuedDpcs)
return VERR_NOT_SUPPORTED;
pArgs = (PRTMPARGS)ExAllocatePoolWithTag(NonPagedPool, MAXIMUM_PROCESSORS*sizeof(KDPC) + sizeof(RTMPARGS), (ULONG)'RTMp');
if (!pArgs)
return VERR_NO_MEMORY;
pArgs->pfnWorker = pfnWorker;
pArgs->pvUser1 = pvUser1;
pArgs->pvUser2 = pvUser2;
pArgs->idCpu = NIL_RTCPUID;
pArgs->idCpu2 = NIL_RTCPUID;
pArgs->cHits = 0;
pArgs->cRefs = 1;
paExecCpuDpcs = (KDPC *)(pArgs + 1);
if (enmCpuid == RT_NT_CPUID_SPECIFIC)
{
KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
KeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
KeSetTargetProcessorDpc(&paExecCpuDpcs[0], (int)idCpu);
pArgs->idCpu = idCpu;
}
else if (enmCpuid == RT_NT_CPUID_SPECIFIC)
{
KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
KeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
KeSetTargetProcessorDpc(&paExecCpuDpcs[0], (int)idCpu);
pArgs->idCpu = idCpu;
KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
KeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
KeSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
pArgs->idCpu2 = idCpu2;
}
else
{
for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
{
KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
KeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
KeSetTargetProcessorDpc(&paExecCpuDpcs[i], i);
}
}
/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
*/
KIRQL oldIrql;
KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
/*
* We cannot do other than assume a 1:1 relationship between the
* affinity mask and the process despite the warnings in the docs.
* If someone knows a better way to get this done, please let bird know.
*/
ASMCompilerBarrier(); /* paranoia */
if (enmCpuid == RT_NT_CPUID_SPECIFIC)
{
ASMAtomicIncS32(&pArgs->cRefs);
BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
Assert(ret);
}
else if (enmCpuid == RT_NT_CPUID_PAIR)
{
ASMAtomicIncS32(&pArgs->cRefs);
BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
Assert(ret);
ASMAtomicIncS32(&pArgs->cRefs);
ret = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
Assert(ret);
}
else
{
unsigned iSelf = KeGetCurrentProcessorNumber();
for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
{
if ( (i != iSelf)
&& (Mask & RT_BIT_64(i)))
{
ASMAtomicIncS32(&pArgs->cRefs);
BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
Assert(ret);
}
}
if (enmCpuid != RT_NT_CPUID_OTHERS)
pfnWorker(iSelf, pvUser1, pvUser2);
}
KeLowerIrql(oldIrql);
/* Flush all DPCs and wait for completion. (can take long!) */
/** @todo Consider changing this to an active wait using some atomic inc/dec
* stuff (and check for the current cpu above in the specific case). */
/** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
* executed. Seen pArgs being freed while some CPU was using it before
* cRefs was added. */
g_pfnrtNtKeFlushQueuedDpcs();
if (pcHits)
*pcHits = pArgs->cHits;
/* Dereference the argument structure. */
int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
Assert(cRefs >= 0);
if (cRefs == 0)
ExFreePool(pArgs);
return VINF_SUCCESS;
}
/**
* @callback_method_impl{FNRTONCE,
* Resolves dynamic imports and initializes globals.}
*/
static DECLCALLBACK(int32_t) rtMpWinInitOnce(void *pvUser)
{
RT_NOREF(pvUser);
Assert(g_WinOsInfoEx.dwOSVersionInfoSize != 0);
Assert(g_hModKernel32 != NULL);
/*
* Resolve dynamic APIs.
*/
#define RESOLVE_API(a_szMod, a_FnName) \
do { \
RT_CONCAT(g_pfn,a_FnName) = (decltype(a_FnName) *)GetProcAddress(g_hModKernel32, #a_FnName); \
} while (0)
RESOLVE_API("kernel32.dll", GetMaximumProcessorCount);
//RESOLVE_API("kernel32.dll", GetActiveProcessorCount); - slow :/
RESOLVE_API("kernel32.dll", GetCurrentProcessorNumber);
RESOLVE_API("kernel32.dll", GetCurrentProcessorNumberEx);
RESOLVE_API("kernel32.dll", GetLogicalProcessorInformation);
RESOLVE_API("kernel32.dll", GetLogicalProcessorInformationEx);
/*
* Reset globals.
*/
for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx); i++)
g_aidRtMpWinByCpuSetIdx[i] = NIL_RTCPUID;
for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpWinCpuGroups); idxGroup++)
{
g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = 0;
g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = 0;
for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
}
/*
* Query group information, partitioning CPU IDs and CPU set indexes.
*
* We ASSUME the GroupInfo index is the same as the group number.
*
* We CANNOT ASSUME that the kernel CPU indexes are assigned in any given
* way, though they usually are in group order by active processor. So,
* we do that to avoid trouble. We must use information provided thru GIP
* if we want the kernel CPU set indexes. Even there, the inactive CPUs
* wont have sensible indexes. Sigh.
*
* We try to assign IDs to inactive CPUs in the same manner as mp-r0drv-nt.cpp
*
* Note! We will die (AssertFatal) if there are too many processors!
*/
union
{
SYSTEM_INFO SysInfo;
SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Info;
uint8_t abPaddingG[ sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
+ sizeof(PROCESSOR_GROUP_INFO) * RTCPUSET_MAX_CPUS];
uint8_t abPaddingC[ sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
+ (sizeof(PROCESSOR_RELATIONSHIP) + sizeof(GROUP_AFFINITY))
* RTCPUSET_MAX_CPUS];
} uBuf;
if (g_pfnGetLogicalProcessorInformationEx)
{
/* Query the information. */
DWORD cbData = sizeof(uBuf);
AssertFatalMsg(g_pfnGetLogicalProcessorInformationEx(RelationGroup, &uBuf.Info, &cbData) != FALSE,
("last error = %u, cbData = %u (in %u)\n", GetLastError(), cbData, sizeof(uBuf)));
AssertFatalMsg(uBuf.Info.Relationship == RelationGroup,
("Relationship = %u, expected %u!\n", uBuf.Info.Relationship, RelationGroup));
AssertFatalMsg(uBuf.Info.Group.MaximumGroupCount <= RT_ELEMENTS(g_aRtMpWinCpuGroups),
("MaximumGroupCount is %u, we only support up to %u!\n",
uBuf.Info.Group.MaximumGroupCount, RT_ELEMENTS(g_aRtMpWinCpuGroups)));
AssertMsg(uBuf.Info.Group.MaximumGroupCount == uBuf.Info.Group.ActiveGroupCount, /* 2nd assumption mentioned above. */
("%u vs %u\n", uBuf.Info.Group.MaximumGroupCount, uBuf.Info.Group.ActiveGroupCount));
AssertFatal(uBuf.Info.Group.MaximumGroupCount >= uBuf.Info.Group.ActiveGroupCount);
g_cRtMpWinMaxCpuGroups = uBuf.Info.Group.MaximumGroupCount;
/* Count max cpus (see mp-r0drv0-nt.cpp) why we don't use GetMaximumProcessorCount(ALL). */
uint32_t idxGroup;
g_cRtMpWinMaxCpus = 0;
for (idxGroup = 0; idxGroup < uBuf.Info.Group.ActiveGroupCount; idxGroup++)
g_cRtMpWinMaxCpus += uBuf.Info.Group.GroupInfo[idxGroup].MaximumProcessorCount;
/* Process the active groups. */
uint32_t cActive = 0;
uint32_t cInactive = 0;
uint32_t idxCpu = 0;
uint32_t idxCpuSetNextInactive = g_cRtMpWinMaxCpus - 1;
for (idxGroup = 0; idxGroup < uBuf.Info.Group.ActiveGroupCount; idxGroup++)
{
PROCESSOR_GROUP_INFO const *pGroupInfo = &uBuf.Info.Group.GroupInfo[idxGroup];
g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = pGroupInfo->MaximumProcessorCount;
g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = pGroupInfo->ActiveProcessorCount;
for (uint32_t idxMember = 0; idxMember < pGroupInfo->MaximumProcessorCount; idxMember++)
{
if (pGroupInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
{
g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
g_aidRtMpWinByCpuSetIdx[idxCpu] = idxCpu;
idxCpu++;
cActive++;
}
else
{
if (idxCpuSetNextInactive >= idxCpu)
{
g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
g_aidRtMpWinByCpuSetIdx[idxCpuSetNextInactive] = idxCpuSetNextInactive;
idxCpuSetNextInactive--;
}
cInactive++;
}
}
}
g_cRtMpWinActiveCpus = cActive;
Assert(cActive + cInactive <= g_cRtMpWinMaxCpus);
Assert(idxCpu <= idxCpuSetNextInactive + 1);
Assert(idxCpu <= g_cRtMpWinMaxCpus);
/* Just in case the 2nd assumption doesn't hold true and there are inactive groups. */
for (; idxGroup < uBuf.Info.Group.MaximumGroupCount; idxGroup++)
{
DWORD cMaxMembers = g_pfnGetMaximumProcessorCount(idxGroup);
g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = cMaxMembers;
g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = 0;
for (uint32_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
{
if (idxCpuSetNextInactive >= idxCpu)
{
g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
g_aidRtMpWinByCpuSetIdx[idxCpuSetNextInactive] = idxCpuSetNextInactive;
idxCpuSetNextInactive--;
}
cInactive++;
}
}
Assert(cActive + cInactive <= g_cRtMpWinMaxCpus);
Assert(idxCpu <= idxCpuSetNextInactive + 1);
}
else
{
/* Legacy: */
GetSystemInfo(&uBuf.SysInfo);
g_cRtMpWinMaxCpuGroups = 1;
g_cRtMpWinMaxCpus = uBuf.SysInfo.dwNumberOfProcessors;
g_aRtMpWinCpuGroups[0].cMaxCpus = uBuf.SysInfo.dwNumberOfProcessors;
g_aRtMpWinCpuGroups[0].cActiveCpus = uBuf.SysInfo.dwNumberOfProcessors;
for (uint32_t idxMember = 0; idxMember < uBuf.SysInfo.dwNumberOfProcessors; idxMember++)
{
g_aRtMpWinCpuGroups[0].aidxCpuSetMembers[idxMember] = idxMember;
g_aidRtMpWinByCpuSetIdx[idxMember] = idxMember;
}
}
AssertFatalMsg(g_cRtMpWinMaxCpus <= RTCPUSET_MAX_CPUS,
("g_cRtMpWinMaxCpus=%u (%#x); RTCPUSET_MAX_CPUS=%u\n", g_cRtMpWinMaxCpus, g_cRtMpWinMaxCpus, RTCPUSET_MAX_CPUS));
g_cbRtMpWinGrpRelBuf = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
+ (g_cRtMpWinMaxCpuGroups + 2) * sizeof(PROCESSOR_GROUP_INFO);
/*
* Get information about cores.
*
* Note! This will only give us info about active processors according to
* MSDN, we'll just have to hope that CPUs aren't hotplugged after we
* initialize here (or that the API consumers doesn't care too much).
*/
/** @todo A hot CPU plug event would be nice. */
g_cRtMpWinMaxCpuCores = g_cRtMpWinMaxCpus;
if (g_pfnGetLogicalProcessorInformationEx)
{
/* Query the information. */
DWORD cbData = sizeof(uBuf);
AssertFatalMsg(g_pfnGetLogicalProcessorInformationEx(RelationProcessorCore, &uBuf.Info, &cbData) != FALSE,
("last error = %u, cbData = %u (in %u)\n", GetLastError(), cbData, sizeof(uBuf)));
g_cRtMpWinMaxCpuCores = 0;
for (uint32_t off = 0; off < cbData; )
{
SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pCur = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)&uBuf.abPaddingG[off];
AssertFatalMsg(pCur->Relationship == RelationProcessorCore,
("off = %#x, Relationship = %u, expected %u!\n", off, pCur->Relationship, RelationProcessorCore));
g_cRtMpWinMaxCpuCores++;
off += pCur->Size;
}
#if ARCH_BITS == 32
if (g_cRtMpWinMaxCpuCores > g_cRtMpWinMaxCpus)
{
/** @todo WOW64 trouble where the emulation environment has folded the high
* processor masks (63..32) into the low (31..0), hiding some
* processors from us. Currently we don't deal with that. */
g_cRtMpWinMaxCpuCores = g_cRtMpWinMaxCpus;
}
else
AssertStmt(g_cRtMpWinMaxCpuCores > 0, g_cRtMpWinMaxCpuCores = g_cRtMpWinMaxCpus);
#else
AssertStmt(g_cRtMpWinMaxCpuCores > 0 && g_cRtMpWinMaxCpuCores <= g_cRtMpWinMaxCpus,
g_cRtMpWinMaxCpuCores = g_cRtMpWinMaxCpus);
#endif
}
else
{
/*
* Sadly, on XP and Server 2003, even if the API is present, it does not tell us
* how many physical cores there are (any package will look like a single core).
* That is worse than not using the API at all, so just skip it unless it's Vista+.
*/
if ( g_pfnGetLogicalProcessorInformation
&& g_WinOsInfoEx.dwPlatformId == VER_PLATFORM_WIN32_NT
&& g_WinOsInfoEx.dwMajorVersion >= 6)
{
/* Query the info. */
DWORD cbSysProcInfo = _4K;
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION paSysInfo = NULL;
BOOL fRc = FALSE;
do
{
cbSysProcInfo = RT_ALIGN_32(cbSysProcInfo, 256);
void *pv = RTMemRealloc(paSysInfo, cbSysProcInfo);
if (!pv)
break;
paSysInfo = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)pv;
fRc = g_pfnGetLogicalProcessorInformation(paSysInfo, &cbSysProcInfo);
} while (!fRc && GetLastError() == ERROR_INSUFFICIENT_BUFFER);
if (fRc)
{
/* Count the cores in the result. */
g_cRtMpWinMaxCpuCores = 0;
uint32_t i = cbSysProcInfo / sizeof(paSysInfo[0]);
while (i-- > 0)
if (paSysInfo[i].Relationship == RelationProcessorCore)
g_cRtMpWinMaxCpuCores++;
AssertStmt(g_cRtMpWinMaxCpuCores > 0 && g_cRtMpWinMaxCpuCores <= g_cRtMpWinMaxCpus,
g_cRtMpWinMaxCpuCores = g_cRtMpWinMaxCpus);
}
RTMemFree(paSysInfo);
}
}
return VINF_SUCCESS;
}
