RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
RTMPARGS Args;
RT_ASSERT_INTS_ON();
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = RTMpCpuId();
Args.cHits = 0;
/* The caller is supposed to have disabled preemption, but take no chances. */
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
RTThreadPreemptDisable(&PreemptState);
RTSOLCPUSET CpuSet;
for (int i = 0; i < IPRT_SOL_SET_WORDS; i++)
CpuSet.auCpus[0] = (ulong_t)-1L;
BT_CLEAR(CpuSet.auCpus, RTMpCpuId());
rtMpSolCrossCall(&CpuSet, rtMpSolOnOtherCpusWrapper, &Args);
RTThreadPreemptRestore(&PreemptState);
return VINF_SUCCESS;
}
RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
int rc;
RTMPARGS Args;
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0)
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
#endif
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = NIL_RTCPUID;
Args.cHits = 0;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
rc = on_each_cpu(rtmpLinuxWrapper, &Args, 1 /* wait */);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0)
rc = on_each_cpu(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
#else /* older kernels */
RTThreadPreemptDisable(&PreemptState);
rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
local_irq_disable();
rtmpLinuxWrapper(&Args);
local_irq_enable();
RTThreadPreemptRestore(&PreemptState);
#endif /* older kernels */
Assert(rc == 0); NOREF(rc);
return VINF_SUCCESS;
}
RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
IPRT_LINUX_SAVE_EFL_AC();
int rc;
RTMPARGS Args;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = NIL_RTCPUID;
Args.cHits = 0;
RTThreadPreemptDisable(&PreemptState);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
rc = smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
#else /* older kernels */
rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
#endif /* older kernels */
RTThreadPreemptRestore(&PreemptState);
Assert(rc == 0); NOREF(rc);
IPRT_LINUX_RESTORE_EFL_AC();
return VINF_SUCCESS;
}
RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
RTMPARGS Args;
RT_ASSERT_INTS_ON();
if (idCpu >= ncpus)
return VERR_CPU_NOT_FOUND;
if (RT_UNLIKELY(!RTMpIsCpuOnline(idCpu)))
return RTMpIsCpuPresent(idCpu) ? VERR_CPU_OFFLINE : VERR_CPU_NOT_FOUND;
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = idCpu;
Args.cHits = 0;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
RTThreadPreemptDisable(&PreemptState);
RTSOLCPUSET CpuSet;
for (int i = 0; i < IPRT_SOL_SET_WORDS; i++)
CpuSet.auCpus[i] = 0;
BT_SET(CpuSet.auCpus, idCpu);
rtMpSolCrossCall(&CpuSet, rtMpSolOnSpecificCpuWrapper, &Args);
RTThreadPreemptRestore(&PreemptState);
Assert(ASMAtomicUoReadU32(&Args.cHits) <= 1);
return ASMAtomicUoReadU32(&Args.cHits) == 1
? VINF_SUCCESS
: VERR_CPU_NOT_FOUND;
}
RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
IPRT_LINUX_SAVE_EFL_AC();
int rc;
RTMPARGS Args;
RTCPUSET OnlineSet;
RTCPUID idCpu;
uint32_t cLoops;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = NIL_RTCPUID;
Args.cHits = 0;
RTThreadPreemptDisable(&PreemptState);
RTMpGetOnlineSet(&OnlineSet);
Args.pWorkerSet = &OnlineSet;
idCpu = RTMpCpuId();
if (RTCpuSetCount(&OnlineSet) > 1)
{
/* Fire the function on all other CPUs without waiting for completion. */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* wait */);
#else
rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* retry */, 0 /* wait */);
#endif
Assert(!rc); NOREF(rc);
}
/* Fire the function on this CPU. */
Args.pfnWorker(idCpu, Args.pvUser1, Args.pvUser2);
RTCpuSetDel(Args.pWorkerSet, idCpu);
/* Wait for all of them finish. */
cLoops = 64000;
while (!RTCpuSetIsEmpty(Args.pWorkerSet))
{
/* Periodically check if any CPU in the wait set has gone offline, if so update the wait set. */
if (!cLoops--)
{
RTCPUSET OnlineSetNow;
RTMpGetOnlineSet(&OnlineSetNow);
RTCpuSetAnd(Args.pWorkerSet, &OnlineSetNow);
cLoops = 64000;
}
ASMNopPause();
}
RTThreadPreemptRestore(&PreemptState);
IPRT_LINUX_RESTORE_EFL_AC();
return VINF_SUCCESS;
}
/**
* Solaris callback function for Mp event notification.
*
* @param CpuState The current event/state of the CPU.
* @param iCpu Which CPU is this event fore.
* @param pvArg Ignored.
*
* @remarks This function assumes index == RTCPUID.
* @returns Solaris error code.
*/
static int rtMpNotificationCpuEvent(cpu_setup_t CpuState, int iCpu, void *pvArg)
{
RTMPEVENT enmMpEvent;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
RTThreadPreemptDisable(&PreemptState);
/*
* Update our CPU set structures first regardless of whether we've been
* scheduled on the right CPU or not, this is just atomic accounting.
*/
if (CpuState == CPU_ON)
{
enmMpEvent = RTMPEVENT_ONLINE;
RTCpuSetAdd(&g_rtMpSolCpuSet, iCpu);
}
else if (CpuState == CPU_OFF)
{
enmMpEvent = RTMPEVENT_OFFLINE;
RTCpuSetDel(&g_rtMpSolCpuSet, iCpu);
}
else
return 0;
/*
* Since we don't absolutely need to do CPU bound code in any of the CPU offline
* notification hooks, run it on the current CPU. Scheduling a callback to execute
* on the CPU going offline at this point is too late and will not work reliably.
*/
bool fRunningOnTargetCpu = iCpu == RTMpCpuId();
if ( fRunningOnTargetCpu == true
|| enmMpEvent == RTMPEVENT_OFFLINE)
{
rtMpNotificationDoCallbacks(enmMpEvent, iCpu);
}
else
{
/*
* We're not on the target CPU, schedule (synchronous) the event notification callback
* to run on the target CPU i.e. the CPU that was online'd.
*/
RTMPARGS Args;
RT_ZERO(Args);
Args.pvUser1 = &enmMpEvent;
Args.pvUser2 = NULL;
Args.idCpu = iCpu;
RTMpOnSpecific(iCpu, rtMpNotificationSolOnCurrentCpu, &Args, NULL /* pvIgnored1 */);
}
RTThreadPreemptRestore(&PreemptState);
NOREF(pvArg);
return 0;
}
RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
int rc;
RTMPARGS Args;
RTSOLCPUSET CpuSet;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = idCpu1;
Args.idCpu2 = idCpu2;
Args.cHits = 0;
for (int i = 0; i < IPRT_SOL_SET_WORDS; i++)
CpuSet.auCpus[i] = 0;
BT_SET(CpuSet.auCpus, idCpu1);
BT_SET(CpuSet.auCpus, idCpu2);
/*
* Check that both CPUs are online before doing the broadcast call.
*/
RTThreadPreemptDisable(&PreemptState);
if ( RTMpIsCpuOnline(idCpu1)
&& RTMpIsCpuOnline(idCpu2))
{
rtMpSolCrossCall(&CpuSet, rtMpSolOnPairCpuWrapper, &Args);
Assert(Args.cHits <= 2);
if (Args.cHits == 2)
rc = VINF_SUCCESS;
else if (Args.cHits == 1)
rc = VERR_NOT_ALL_CPUS_SHOWED;
else if (Args.cHits == 0)
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_IPE_1;
}
/*
* A CPU must be present to be considered just offline.
*/
else if ( RTMpIsCpuPresent(idCpu1)
&& RTMpIsCpuPresent(idCpu2))
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_NOT_FOUND;
RTThreadPreemptRestore(&PreemptState);
return rc;
}
RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
IPRT_LINUX_SAVE_EFL_AC();
int rc;
RTMPARGS Args;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = idCpu;
Args.cHits = 0;
if (!RTMpIsCpuPossible(idCpu))
return VERR_CPU_NOT_FOUND;
RTThreadPreemptDisable(&PreemptState);
if (idCpu != RTMpCpuId())
{
if (RTMpIsCpuOnline(idCpu))
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 1 /* wait */);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
#else /* older kernels */
rc = smp_call_function(rtmpOnSpecificLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
#endif /* older kernels */
Assert(rc == 0);
rc = Args.cHits ? VINF_SUCCESS : VERR_CPU_OFFLINE;
}
else
rc = VERR_CPU_OFFLINE;
}
else
{
rtmpLinuxWrapper(&Args);
rc = VINF_SUCCESS;
}
RTThreadPreemptRestore(&PreemptState);;
NOREF(rc);
IPRT_LINUX_RESTORE_EFL_AC();
return rc;
}
RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
RTMPARGS Args;
RT_ASSERT_INTS_ON();
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = NIL_RTCPUID;
Args.cHits = 0;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
RTThreadPreemptDisable(&PreemptState);
RTSOLCPUSET CpuSet;
for (int i = 0; i < IPRT_SOL_SET_WORDS; i++)
CpuSet.auCpus[i] = (ulong_t)-1L;
rtMpSolCrossCall(&CpuSet, rtMpSolOnAllCpuWrapper, &Args);
RTThreadPreemptRestore(&PreemptState);
return VINF_SUCCESS;
}
/**
* Service request callback function.
*
* @returns VBox status code.
* @param pSession The caller's session.
* @param u64Arg 64-bit integer argument.
* @param pReqHdr The request header. Input / Output. Optional.
*/
DECLEXPORT(int) TSTR0ThreadPreemptionSrvReqHandler(PSUPDRVSESSION pSession, uint32_t uOperation,
uint64_t u64Arg, PSUPR0SERVICEREQHDR pReqHdr)
{
NOREF(pSession);
if (u64Arg)
return VERR_INVALID_PARAMETER;
if (!VALID_PTR(pReqHdr))
return VERR_INVALID_PARAMETER;
char *pszErr = (char *)(pReqHdr + 1);
size_t cchErr = pReqHdr->cbReq - sizeof(*pReqHdr);
if (cchErr < 32 || cchErr >= 0x10000)
return VERR_INVALID_PARAMETER;
*pszErr = '\0';
/*
* The big switch.
*/
switch (uOperation)
{
case TSTR0THREADPREMEPTION_SANITY_OK:
break;
case TSTR0THREADPREMEPTION_SANITY_FAILURE:
RTStrPrintf(pszErr, cchErr, "!42failure42%1024s", "");
break;
case TSTR0THREADPREMEPTION_BASIC:
{
if (!ASMIntAreEnabled())
RTStrPrintf(pszErr, cchErr, "!Interrupts disabled");
else if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
RTStrPrintf(pszErr, cchErr, "!RTThreadPreemptIsEnabled returns false by default");
else
{
RTTHREADPREEMPTSTATE State = RTTHREADPREEMPTSTATE_INITIALIZER;
RTThreadPreemptDisable(&State);
if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
RTStrPrintf(pszErr, cchErr, "!RTThreadPreemptIsEnabled returns true after RTThreadPreemptDisable");
else if (!ASMIntAreEnabled())
RTStrPrintf(pszErr, cchErr, "!Interrupts disabled");
RTThreadPreemptRestore(&State);
}
break;
}
case TSTR0THREADPREMEPTION_IS_PENDING:
{
RTTHREADPREEMPTSTATE State = RTTHREADPREEMPTSTATE_INITIALIZER;
RTThreadPreemptDisable(&State);
if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
{
if (ASMIntAreEnabled())
{
uint64_t u64StartTS = RTTimeNanoTS();
uint64_t u64StartSysTS = RTTimeSystemNanoTS();
uint64_t cLoops = 0;
uint64_t cNanosSysElapsed;
uint64_t cNanosElapsed;
bool fPending;
do
{
fPending = RTThreadPreemptIsPending(NIL_RTTHREAD);
cNanosElapsed = RTTimeNanoTS() - u64StartTS;
cNanosSysElapsed = RTTimeSystemNanoTS() - u64StartSysTS;
cLoops++;
} while ( !fPending
&& cNanosElapsed < UINT64_C(2)*1000U*1000U*1000U
&& cNanosSysElapsed < UINT64_C(2)*1000U*1000U*1000U
&& cLoops < 100U*_1M);
if (!fPending)
RTStrPrintf(pszErr, cchErr, "!Preempt not pending after %'llu loops / %'llu ns / %'llu ns (sys)",
cLoops, cNanosElapsed, cNanosSysElapsed);
else if (cLoops == 1)
RTStrPrintf(pszErr, cchErr, "!cLoops=1\n");
else
RTStrPrintf(pszErr, cchErr, "RTThreadPreemptIsPending returned true after %'llu loops / %'llu ns / %'llu ns (sys)",
cLoops, cNanosElapsed, cNanosSysElapsed);
}
else
RTStrPrintf(pszErr, cchErr, "!Interrupts disabled");
}
else
RTStrPrintf(pszErr, cchErr, "!RTThreadPreemptIsEnabled returns true after RTThreadPreemptDisable");
RTThreadPreemptRestore(&State);
break;
}
case TSTR0THREADPREMEPTION_NESTED:
{
bool const fDefault = RTThreadPreemptIsEnabled(NIL_RTTHREAD);
RTTHREADPREEMPTSTATE State1 = RTTHREADPREEMPTSTATE_INITIALIZER;
RTThreadPreemptDisable(&State1);
if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
{
RTTHREADPREEMPTSTATE State2 = RTTHREADPREEMPTSTATE_INITIALIZER;
RTThreadPreemptDisable(&State2);
if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
{
RTTHREADPREEMPTSTATE State3 = RTTHREADPREEMPTSTATE_INITIALIZER;
RTThreadPreemptDisable(&State3);
if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
RTStrPrintf(pszErr, cchErr, "!RTThreadPreemptIsEnabled returns true after 3rd RTThreadPreemptDisable");
RTThreadPreemptRestore(&State3);
if (RTThreadPreemptIsEnabled(NIL_RTTHREAD) && !*pszErr)
RTStrPrintf(pszErr, cchErr, "!RTThreadPreemptIsEnabled returns true after 1st RTThreadPreemptRestore");
}
else
RTStrPrintf(pszErr, cchErr, "!RTThreadPreemptIsEnabled returns true after 2nd RTThreadPreemptDisable");
RTThreadPreemptRestore(&State2);
if (RTThreadPreemptIsEnabled(NIL_RTTHREAD) && !*pszErr)
RTStrPrintf(pszErr, cchErr, "!RTThreadPreemptIsEnabled returns true after 2nd RTThreadPreemptRestore");
}
else
RTStrPrintf(pszErr, cchErr, "!RTThreadPreemptIsEnabled returns true after 1st RTThreadPreemptDisable");
RTThreadPreemptRestore(&State1);
if (RTThreadPreemptIsEnabled(NIL_RTTHREAD) != fDefault && !*pszErr)
RTStrPrintf(pszErr, cchErr, "!RTThreadPreemptIsEnabled returns false after 3rd RTThreadPreemptRestore");
break;
}
default:
RTStrPrintf(pszErr, cchErr, "!Unknown test #%d", uOperation);
break;
}
/* The error indicator is the '!' in the message buffer. */
return VINF_SUCCESS;
}
RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
IPRT_LINUX_SAVE_EFL_AC();
int rc;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
/*
* Check that both CPUs are online before doing the broadcast call.
*/
RTThreadPreemptDisable(&PreemptState);
if ( RTMpIsCpuOnline(idCpu1)
&& RTMpIsCpuOnline(idCpu2))
{
/*
* Use the smp_call_function variant taking a cpu mask where available,
* falling back on broadcast with filter. Slight snag if one of the
* CPUs is the one we're running on, we must do the call and the post
* call wait ourselves.
*/
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
cpumask_t DstCpuMask;
#endif
RTCPUID idCpuSelf = RTMpCpuId();
bool const fCallSelf = idCpuSelf == idCpu1 || idCpuSelf == idCpu2;
RTMPARGS Args;
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = idCpu1;
Args.idCpu2 = idCpu2;
Args.cHits = 0;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
cpumask_clear(&DstCpuMask);
cpumask_set_cpu(idCpu1, &DstCpuMask);
cpumask_set_cpu(idCpu2, &DstCpuMask);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
cpus_clear(DstCpuMask);
cpu_set(idCpu1, DstCpuMask);
cpu_set(idCpu2, DstCpuMask);
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
rc = 0;
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
rc = smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
rc = smp_call_function_mask(DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
#else /* older kernels */
rc = smp_call_function(rtMpLinuxOnPairWrapper, &Args, 0 /* retry */, !fCallSelf /* wait */);
#endif /* older kernels */
Assert(rc == 0);
/* Call ourselves if necessary and wait for the other party to be done. */
if (fCallSelf)
{
uint32_t cLoops = 0;
rtmpLinuxWrapper(&Args);
while (ASMAtomicReadU32(&Args.cHits) < 2)
{
if ((cLoops & 0x1ff) == 0 && !RTMpIsCpuOnline(idCpuSelf == idCpu1 ? idCpu2 : idCpu1))
break;
cLoops++;
ASMNopPause();
}
}
Assert(Args.cHits <= 2);
if (Args.cHits == 2)
rc = VINF_SUCCESS;
else if (Args.cHits == 1)
rc = VERR_NOT_ALL_CPUS_SHOWED;
else if (Args.cHits == 0)
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_IPE_1;
}
/*
* A CPU must be present to be considered just offline.
*/
else if ( RTMpIsCpuPresent(idCpu1)
&& RTMpIsCpuPresent(idCpu2))
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_NOT_FOUND;
RTThreadPreemptRestore(&PreemptState);;
IPRT_LINUX_RESTORE_EFL_AC();
return rc;
}
/**
* Common worker for the debug and normal APIs.
*
* @returns VINF_SUCCESS if entered successfully.
* @returns rcBusy when encountering a busy critical section in GC/R0.
* @retval VERR_SEM_DESTROYED if the critical section is delete before or
* during the operation.
*
* @param pCritSect The PDM critical section to enter.
* @param rcBusy The status code to return when we're in GC or R0
* and the section is busy.
*/
DECL_FORCE_INLINE(int) pdmCritSectEnter(PPDMCRITSECT pCritSect, int rcBusy, PCRTLOCKVALSRCPOS pSrcPos)
{
Assert(pCritSect->s.Core.cNestings < 8); /* useful to catch incorrect locking */
Assert(pCritSect->s.Core.cNestings >= 0);
/*
* If the critical section has already been destroyed, then inform the caller.
*/
AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC,
("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic),
VERR_SEM_DESTROYED);
/*
* See if we're lucky.
*/
/* NOP ... */
if (pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP)
return VINF_SUCCESS;
RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pCritSect);
/* ... not owned ... */
if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
/* ... or nested. */
if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf)
{
ASMAtomicIncS32(&pCritSect->s.Core.cLockers);
ASMAtomicIncS32(&pCritSect->s.Core.cNestings);
Assert(pCritSect->s.Core.cNestings > 1);
return VINF_SUCCESS;
}
/*
* Spin for a bit without incrementing the counter.
*/
/** @todo Move this to cfgm variables since it doesn't make sense to spin on UNI
* cpu systems. */
int32_t cSpinsLeft = CTX_SUFF(PDMCRITSECT_SPIN_COUNT_);
while (cSpinsLeft-- > 0)
{
if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
ASMNopPause();
/** @todo Should use monitor/mwait on e.g. &cLockers here, possibly with a
cli'ed pendingpreemption check up front using sti w/ instruction fusing
for avoiding races. Hmm ... This is assuming the other party is actually
executing code on another CPU ... which we could keep track of if we
wanted. */
}
#ifdef IN_RING3
/*
* Take the slow path.
*/
NOREF(rcBusy);
return pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
#else
# ifdef IN_RING0
/** @todo If preemption is disabled it means we're in VT-x/AMD-V context
* and would be better off switching out of that while waiting for
* the lock. Several of the locks jumps back to ring-3 just to
* get the lock, the ring-3 code will then call the kernel to do
* the lock wait and when the call return it will call ring-0
* again and resume via in setjmp style. Not very efficient. */
# if 0
if (ASMIntAreEnabled()) /** @todo this can be handled as well by changing
* callers not prepared for longjmp/blocking to
* use PDMCritSectTryEnter. */
{
/*
* Leave HM context while waiting if necessary.
*/
int rc;
if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
{
STAM_REL_COUNTER_ADD(&pCritSect->s.StatContentionRZLock, 1000000);
rc = pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
}
else
{
STAM_REL_COUNTER_ADD(&pCritSect->s.StatContentionRZLock, 1000000000);
PVM pVM = pCritSect->s.CTX_SUFF(pVM);
PVMCPU pVCpu = VMMGetCpu(pVM);
HMR0Leave(pVM, pVCpu);
RTThreadPreemptRestore(NIL_RTTHREAD, XXX);
rc = pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
RTThreadPreemptDisable(NIL_RTTHREAD, XXX);
HMR0Enter(pVM, pVCpu);
}
return rc;
}
# else
/*
* We preemption hasn't been disabled, we can block here in ring-0.
*/
if ( RTThreadPreemptIsEnabled(NIL_RTTHREAD)
&& ASMIntAreEnabled())
return pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
# endif
#endif /* IN_RING0 */
STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock);
/*
* Call ring-3 to acquire the critical section?
*/
if (rcBusy == VINF_SUCCESS)
{
PVM pVM = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM);
PVMCPU pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_ENTER, MMHyperCCToR3(pVM, pCritSect));
}
/*
* Return busy.
*/
LogFlow(("PDMCritSectEnter: locked => R3 (%Rrc)\n", rcBusy));
return rcBusy;
#endif /* !IN_RING3 */
}
/**
* Helper for RTSemSpinMutexTryRequest and RTSemSpinMutexRequest.
*
* This will check the current context and see if it's usui
*
* @returns VINF_SUCCESS or VERR_SEM_BAD_CONTEXT.
* @param pState Output structure.
*/
static int rtSemSpinMutexEnter(RTSEMSPINMUTEXSTATE *pState, RTSEMSPINMUTEXINTERNAL *pThis)
{
#ifndef RT_OS_WINDOWS
RTTHREADPREEMPTSTATE const StateInit = RTTHREADPREEMPTSTATE_INITIALIZER;
#endif
int rc = VINF_SUCCESS;
/** @todo Later #1: When entering in interrupt context and we're not able to
* wake up threads from it, we could try switch the lock into pure
* spinlock mode. This would require that there are no other threads
* currently waiting on it and that the RTSEMSPINMUTEX_FLAGS_IRQ_SAFE
* flag is set.
*
* Later #2: Similarly, it is possible to turn on the
* RTSEMSPINMUTEX_FLAGS_IRQ_SAFE at run time if we manage to grab the
* semaphore ownership at interrupt time. We might want to try delay the
* RTSEMSPINMUTEX_FLAGS_IRQ_SAFE even, since we're fine if we get it...
*/
#ifdef RT_OS_WINDOWS
/*
* NT: IRQL <= DISPATCH_LEVEL for waking up threads; IRQL < DISPATCH_LEVEL for sleeping.
*/
pState->PreemptState.uchOldIrql = KeGetCurrentIrql();
if (pState->PreemptState.uchOldIrql > DISPATCH_LEVEL)
return VERR_SEM_BAD_CONTEXT;
if (pState->PreemptState.uchOldIrql >= DISPATCH_LEVEL)
pState->fSpin = true;
else
{
pState->fSpin = false;
KeRaiseIrql(DISPATCH_LEVEL, &pState->PreemptState.uchOldIrql);
Assert(pState->PreemptState.uchOldIrql < DISPATCH_LEVEL);
}
#elif defined(RT_OS_SOLARIS)
/*
* Solaris: RTSemEventSignal will do bad stuff on S10 if interrupts are disabled.
*/
if (!ASMIntAreEnabled())
return VERR_SEM_BAD_CONTEXT;
pState->fSpin = !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
if (RTThreadIsInInterrupt(NIL_RTTHREAD))
{
if (!(pThis->fFlags & RTSEMSPINMUTEX_FLAGS_IRQ_SAFE))
rc = VINF_SEM_BAD_CONTEXT; /* Try, but owner might be interrupted. */
pState->fSpin = true;
}
pState->PreemptState = StateInit;
RTThreadPreemptDisable(&pState->PreemptState);
#elif defined(RT_OS_LINUX) || defined(RT_OS_OS2)
/*
* OSes on which RTSemEventSignal can be called from any context.
*/
pState->fSpin = !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
if (RTThreadIsInInterrupt(NIL_RTTHREAD))
{
if (!(pThis->fFlags & RTSEMSPINMUTEX_FLAGS_IRQ_SAFE))
rc = VINF_SEM_BAD_CONTEXT; /* Try, but owner might be interrupted. */
pState->fSpin = true;
}
pState->PreemptState = StateInit;
RTThreadPreemptDisable(&pState->PreemptState);
#else /* PORTME: Check for context where we cannot wake up threads. */
/*
* Default: ASSUME thread can be woken up if interrupts are enabled and
* we're not in an interrupt context.
* ASSUME that we can go to sleep if preemption is enabled.
*/
if ( RTThreadIsInInterrupt(NIL_RTTHREAD)
|| !ASMIntAreEnabled())
return VERR_SEM_BAD_CONTEXT;
pState->fSpin = !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
pState->PreemptState = StateInit;
RTThreadPreemptDisable(&pState->PreemptState);
#endif
/*
* Disable interrupts if necessary.
*/
pState->fValidFlags = !!(pThis->fFlags & RTSEMSPINMUTEX_FLAGS_IRQ_SAFE);
if (pState->fValidFlags)
pState->fSavedFlags = ASMIntDisableFlags();
else
pState->fSavedFlags = 0;
return rc;
}