/**
* Notification wrapper that updates CPU states and invokes our notification
* callbacks.
*
* @param idCpu The CPU Id.
* @param pvUser1 Pointer to the notifier_block (unused).
* @param pvUser2 The notification event.
* @remarks This can be invoked in interrupt context.
*/
static DECLCALLBACK(void) rtMpNotificationLinuxOnCurrentCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
{
unsigned long ulNativeEvent = *(unsigned long *)pvUser2;
NOREF(pvUser1);
AssertRelease(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
AssertReleaseMsg(idCpu == RTMpCpuId(), /* ASSUMES iCpu == RTCPUID */
("idCpu=%u RTMpCpuId=%d ApicId=%d\n", idCpu, RTMpCpuId(), ASMGetApicId() ));
switch (ulNativeEvent)
{
# ifdef CPU_DOWN_FAILED
case CPU_DOWN_FAILED:
# if defined(CPU_TASKS_FROZEN) && defined(CPU_DOWN_FAILED_FROZEN)
case CPU_DOWN_FAILED_FROZEN:
# endif
# endif
case CPU_ONLINE:
# if defined(CPU_TASKS_FROZEN) && defined(CPU_ONLINE_FROZEN)
case CPU_ONLINE_FROZEN:
# endif
rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, idCpu);
break;
# ifdef CPU_DOWN_PREPARE
case CPU_DOWN_PREPARE:
# if defined(CPU_TASKS_FROZEN) && defined(CPU_DOWN_PREPARE_FROZEN)
case CPU_DOWN_PREPARE_FROZEN:
# endif
rtMpNotificationDoCallbacks(RTMPEVENT_OFFLINE, idCpu);
break;
# endif
}
}
RTDECL(RTCPUID) RTMpCpuId(void)
{
RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
RTMPWIN_UPDATE_GIP_GLOBALS();
PROCESSOR_NUMBER ProcNum;
ProcNum.Group = 0;
ProcNum.Number = 0xff;
if (g_pfnGetCurrentProcessorNumberEx)
g_pfnGetCurrentProcessorNumberEx(&ProcNum);
else if (g_pfnGetCurrentProcessorNumber)
{
DWORD iCpu = g_pfnGetCurrentProcessorNumber();
Assert(iCpu < g_cRtMpWinMaxCpus);
ProcNum.Number = iCpu;
}
else
{
#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
ProcNum.Number = ASMGetApicId();
#else
# error "Not ported to this architecture."
return NIL_RTCPUID;
#endif
}
#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
#else
return RTMpSetIndexFromCpuGroupMember(ProcNum.Group, ProcNum.Number);
#endif
}
RTDECL(bool) RTThreadPreemptIsEnabled(RTTHREAD hThread)
{
Assert(hThread == NIL_RTTHREAD);
int32_t c = g_acPreemptDisabled[ASMGetApicId()];
AssertMsg(c >= 0 && c < 32, ("%d\n", c));
return c == 0
&& ASMIntAreEnabled();
}
RTDECL(RTCPUID) RTMpCpuId(void)
{
#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64) || defined(RT_ARCH_SPARC) || defined(RT_ARCH_SPARC64)
return ASMGetApicId();
#else
# error "Not ported to this architecture."
return NIL_RTAPICID;
#endif
}
RTDECL(void) RTThreadPreemptDisable(PRTTHREADPREEMPTSTATE pState)
{
AssertPtr(pState);
Assert(pState->u32Reserved == 0);
/* No preemption on OS/2, so do our own accounting. */
int32_t c = ASMAtomicIncS32(&g_acPreemptDisabled[ASMGetApicId()]);
AssertMsg(c > 0 && c < 32, ("%d\n", c));
pState->u32Reserved = c;
RT_ASSERT_PREEMPT_CPUID_DISABLE(pState);
}
RTDECL(void) RTThreadPreemptRestore(PRTTHREADPREEMPTSTATE pState)
{
AssertPtr(pState);
AssertMsg(pState->u32Reserved > 0 && pState->u32Reserved < 32, ("%d\n", pState->u32Reserved));
RT_ASSERT_PREEMPT_CPUID_RESTORE(pState);
/* No preemption on OS/2, so do our own accounting. */
int32_t volatile *pc = &g_acPreemptDisabled[ASMGetApicId()];
AssertMsg(pState->u32Reserved == (uint32_t)*pc, ("uchDummy=%d *pc=%d \n", pState->u32Reserved, *pc));
ASMAtomicUoWriteS32(pc, pState->u32Reserved - 1);
pState->u32Reserved = 0;
}
/**
* Internal worker for getting the GIP CPU array index for the calling CPU.
*
* @returns Index into SUPGLOBALINFOPAGE::aCPUs or UINT16_MAX.
* @param pGip The GIP.
*/
DECLINLINE(uint16_t) supGetGipCpuIndex(PSUPGLOBALINFOPAGE pGip)
{
uint16_t iGipCpu;
#ifdef IN_RING3
if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS)
{
/* Storing the IDTR is normally very fast. */
uint16_t cbLim = ASMGetIdtrLimit();
uint16_t iCpuSet = cbLim - 256 * (ARCH_BITS == 64 ? 16 : 8);
iCpuSet &= RTCPUSET_MAX_CPUS - 1;
iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
}
else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS)
{
/* RDTSCP gives us what need need and more. */
uint32_t iCpuSet;
ASMReadTscWithAux(&iCpuSet);
iCpuSet &= RTCPUSET_MAX_CPUS - 1;
iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
}
else
{
/* Get APIC ID via the slow CPUID instruction. */
uint8_t idApic = ASMGetApicId();
iGipCpu = pGip->aiCpuFromApicId[idApic];
}
#elif defined(IN_RING0)
/* Ring-0: Use use RTMpCpuId() (disables cli to avoid host OS assertions about unsafe CPU number usage). */
RTCCUINTREG uFlags = ASMIntDisableFlags();
int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
if (RT_LIKELY((unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)))
iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
else
iGipCpu = UINT16_MAX;
ASMSetFlags(uFlags);
# elif defined(IN_RC)
/* Raw-mode context: We can get the host CPU set index via VMCPU. */
uint32_t iCpuSet = VMMGetCpu(&g_VM)->iHostCpuSet;
if (RT_LIKELY(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)))
iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
else
iGipCpu = UINT16_MAX;
#else
# error "IN_RING3, IN_RC or IN_RING0 must be defined!"
#endif
return iGipCpu;
}
/**
* The slow case for SUPReadTsc where we need to apply deltas.
*
* Must only be called when deltas are applicable, so please do not call it
* directly.
*
* @returns TSC with delta applied.
* @param pGip Pointer to the GIP.
*
* @remarks May be called with interrupts disabled in ring-0! This is why the
* ring-0 code doesn't attempt to figure the delta.
*
* @internal
*/
SUPDECL(uint64_t) SUPReadTscWithDelta(PSUPGLOBALINFOPAGE pGip)
{
uint64_t uTsc;
uint16_t iGipCpu;
AssertCompile(RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS));
AssertCompile(RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx) >= RTCPUSET_MAX_CPUS);
Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO);
/*
* Read the TSC and get the corresponding aCPUs index.
*/
#ifdef IN_RING3
if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS)
{
/* RDTSCP gives us all we need, no loops/cli. */
uint32_t iCpuSet;
uTsc = ASMReadTscWithAux(&iCpuSet);
iCpuSet &= RTCPUSET_MAX_CPUS - 1;
iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
}
else if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS)
{
/* Storing the IDTR is normally very quick, but we need to loop. */
uint32_t cTries = 0;
for (;;)
{
uint16_t cbLim = ASMGetIdtrLimit();
uTsc = ASMReadTSC();
if (RT_LIKELY(ASMGetIdtrLimit() == cbLim))
{
uint16_t iCpuSet = cbLim - 256 * (ARCH_BITS == 64 ? 16 : 8);
iCpuSet &= RTCPUSET_MAX_CPUS - 1;
iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
break;
}
if (cTries >= 16)
{
iGipCpu = UINT16_MAX;
break;
}
cTries++;
}
}
else
{
/* Get APIC ID via the slow CPUID instruction, requires looping. */
uint32_t cTries = 0;
for (;;)
{
uint8_t idApic = ASMGetApicId();
uTsc = ASMReadTSC();
if (RT_LIKELY(ASMGetApicId() == idApic))
{
iGipCpu = pGip->aiCpuFromApicId[idApic];
break;
}
if (cTries >= 16)
{
iGipCpu = UINT16_MAX;
break;
}
cTries++;
}
}
#elif defined(IN_RING0)
/* Ring-0: Use use RTMpCpuId(), no loops. */
RTCCUINTREG uFlags = ASMIntDisableFlags();
int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
if (RT_LIKELY((unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)))
iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
else
iGipCpu = UINT16_MAX;
uTsc = ASMReadTSC();
ASMSetFlags(uFlags);
# elif defined(IN_RC)
/* Raw-mode context: We can get the host CPU set index via VMCPU, no loops. */
RTCCUINTREG uFlags = ASMIntDisableFlags(); /* Are already disable, but play safe. */
uint32_t iCpuSet = VMMGetCpu(&g_VM)->iHostCpuSet;
if (RT_LIKELY(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)))
iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
else
iGipCpu = UINT16_MAX;
uTsc = ASMReadTSC();
ASMSetFlags(uFlags);
#else
# error "IN_RING3, IN_RC or IN_RING0 must be defined!"
#endif
/*
* If the delta is valid, apply it.
*/
if (RT_LIKELY(iGipCpu < pGip->cCpus))
{
int64_t iTscDelta = pGip->aCPUs[iGipCpu].i64TSCDelta;
if (RT_LIKELY(iTscDelta != INT64_MAX))
return uTsc - iTscDelta;
# ifdef IN_RING3
/*
* The delta needs calculating, call supdrv to get the TSC.
*/
int rc = SUPR3ReadTsc(&uTsc, NULL);
if (RT_SUCCESS(rc))
return uTsc;
AssertMsgFailed(("SUPR3ReadTsc -> %Rrc\n", rc));
uTsc = ASMReadTSC();
# endif /* IN_RING3 */
}
/*
* This shouldn't happen, especially not in ring-3 and raw-mode context.
* But if it does, return something that's half useful.
*/
AssertMsgFailed(("iGipCpu=%d (%#x) cCpus=%d fGetGipCpu=%#x\n", iGipCpu, iGipCpu, pGip->cCpus, pGip->fGetGipCpu));
return uTsc;
}
