RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
if (g_pfnrtKeIpiGenericCall)
return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
NIL_RTCPUID, NIL_RTCPUID, NULL);
return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
}
RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
int rc;
AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
return VERR_NOT_SUPPORTED;
/*
* Check that both CPUs are online before doing the broadcast call.
*/
if ( RTMpIsCpuOnline(idCpu1)
&& RTMpIsCpuOnline(idCpu2))
{
/*
* The broadcast IPI isn't quite as bad as it could have been, because
* it looks like windows doesn't synchronize CPUs on the way out, they
* seems to get back to normal work while the pair is still busy.
*/
uint32_t cHits = 0;
if (g_pfnrtKeIpiGenericCall)
rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
else
rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
if (RT_SUCCESS(rc))
{
Assert(cHits <= 2);
if (cHits == 2)
rc = VINF_SUCCESS;
else if (cHits == 1)
rc = VERR_NOT_ALL_CPUS_SHOWED;
else if (cHits == 0)
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_IPE_1;
}
}
/*
* A CPU must be present to be considered just offline.
*/
else if ( RTMpIsCpuPresent(idCpu1)
&& RTMpIsCpuPresent(idCpu2))
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_NOT_FOUND;
return rc;
}
RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
/*
* Don't try mess with an offline CPU.
*/
if (!RTMpIsCpuOnline(idCpu))
return !RTMpIsCpuPossible(idCpu)
? VERR_CPU_NOT_FOUND
: VERR_CPU_OFFLINE;
/*
* Use the broadcast IPI routine if there are no more than two CPUs online,
* or if the current IRQL is unsuitable for KeWaitForSingleObject.
*/
int rc;
uint32_t cHits = 0;
if ( g_pfnrtKeIpiGenericCall
&& ( RTMpGetOnlineCount() <= 2
|| KeGetCurrentIrql() > APC_LEVEL)
)
{
rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
idCpu, NIL_RTCPUID, &cHits);
if (RT_SUCCESS(rc))
{
if (cHits == 1)
return VINF_SUCCESS;
rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
}
return rc;
}
#if 0
rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
if (RT_SUCCESS(rc))
{
if (cHits == 1)
return VINF_SUCCESS;
rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
}
return rc;
#else
/*
* Initialize the argument package and the objects within it.
* The package is referenced counted to avoid unnecessary spinning to
* synchronize cleanup and prevent stack corruption.
*/
PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)ExAllocatePoolWithTag(NonPagedPool, sizeof(*pArgs), (ULONG)'RTMp');
if (!pArgs)
return VERR_NO_MEMORY;
pArgs->cRefs = 2;
pArgs->fExecuting = false;
pArgs->fDone = false;
pArgs->CallbackArgs.pfnWorker = pfnWorker;
pArgs->CallbackArgs.pvUser1 = pvUser1;
pArgs->CallbackArgs.pvUser2 = pvUser2;
pArgs->CallbackArgs.idCpu = idCpu;
pArgs->CallbackArgs.cHits = 0;
pArgs->CallbackArgs.cRefs = 2;
KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
KeSetImportanceDpc(&pArgs->Dpc, HighImportance);
KeSetTargetProcessorDpc(&pArgs->Dpc, (int)idCpu);
/*
* Disable preemption while we check the current processor and inserts the DPC.
*/
KIRQL bOldIrql;
KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
ASMCompilerBarrier(); /* paranoia */
if (RTMpCpuId() == idCpu)
{
/* Just execute the callback on the current CPU. */
pfnWorker(idCpu, pvUser1, pvUser2);
KeLowerIrql(bOldIrql);
ExFreePool(pArgs);
return VINF_SUCCESS;
}
/* Different CPU, so queue it if the CPU is still online. */
if (RTMpIsCpuOnline(idCpu))
{
BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
Assert(fRc);
KeLowerIrql(bOldIrql);
uint64_t const nsRealWaitTS = RTTimeNanoTS();
/*
* Wait actively for a while in case the CPU/thread responds quickly.
*/
uint32_t cLoopsLeft = 0x20000;
while (cLoopsLeft-- > 0)
{
if (pArgs->fDone)
{
rtMpNtOnSpecificRelease(pArgs);
return VINF_SUCCESS;
}
ASMNopPause();
}
/*
* It didn't respond, so wait on the event object, poking the CPU if it's slow.
*/
LARGE_INTEGER Timeout;
Timeout.QuadPart = -10000; /* 1ms */
NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
if (rcNt == STATUS_SUCCESS)
{
rtMpNtOnSpecificRelease(pArgs);
return VINF_SUCCESS;
}
/* If it hasn't respondend yet, maybe poke it and wait some more. */
if (rcNt == STATUS_TIMEOUT)
{
#ifndef IPRT_TARGET_NT4
if ( !pArgs->fExecuting
&& ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalSendSoftwareInterrupt
|| g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalReqestIpiW7Plus
|| g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalReqestIpiPreW7))
RTMpPokeCpu(idCpu);
#endif
Timeout.QuadPart = -1280000; /* 128ms */
rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
if (rcNt == STATUS_SUCCESS)
{
rtMpNtOnSpecificRelease(pArgs);
return VINF_SUCCESS;
}
}
/*
* Something weird is happening, try bail out.
*/
if (KeRemoveQueueDpc(&pArgs->Dpc))
{
ExFreePool(pArgs); /* DPC was still queued, so we can return without further ado. */
LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
}
else
{
/* DPC is running, wait a good while for it to complete. */
LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
Timeout.QuadPart = -30*1000*1000*10; /* 30 seconds */
rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
if (rcNt != STATUS_SUCCESS)
LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
}
rc = RTErrConvertFromNtStatus(rcNt);
}
else
{
/* CPU is offline.*/
KeLowerIrql(bOldIrql);
rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
}
rtMpNtOnSpecificRelease(pArgs);
return rc;
#endif
}
RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
if (g_pfnrtKeIpiGenericCall)
return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper, 0);
return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, 0);
}
