/** * Notification wrapper that updates CPU states and invokes our notification * callbacks. * * @param idCpu The CPU Id. * @param pvUser1 Pointer to the notifier_block (unused). * @param pvUser2 The notification event. * @remarks This can be invoked in interrupt context. */ static DECLCALLBACK(void) rtMpNotificationLinuxOnCurrentCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2) { unsigned long ulNativeEvent = *(unsigned long *)pvUser2; NOREF(pvUser1); AssertRelease(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); AssertReleaseMsg(idCpu == RTMpCpuId(), /* ASSUMES iCpu == RTCPUID */ ("idCpu=%u RTMpCpuId=%d ApicId=%d\n", idCpu, RTMpCpuId(), ASMGetApicId() )); switch (ulNativeEvent) { # ifdef CPU_DOWN_FAILED case CPU_DOWN_FAILED: # if defined(CPU_TASKS_FROZEN) && defined(CPU_DOWN_FAILED_FROZEN) case CPU_DOWN_FAILED_FROZEN: # endif # endif case CPU_ONLINE: # if defined(CPU_TASKS_FROZEN) && defined(CPU_ONLINE_FROZEN) case CPU_ONLINE_FROZEN: # endif rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, idCpu); break; # ifdef CPU_DOWN_PREPARE case CPU_DOWN_PREPARE: # if defined(CPU_TASKS_FROZEN) && defined(CPU_DOWN_PREPARE_FROZEN) case CPU_DOWN_PREPARE_FROZEN: # endif rtMpNotificationDoCallbacks(RTMPEVENT_OFFLINE, idCpu); break; # endif } }
RTDECL(RTCPUID) RTMpCpuId(void) { RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); RTMPWIN_UPDATE_GIP_GLOBALS(); PROCESSOR_NUMBER ProcNum; ProcNum.Group = 0; ProcNum.Number = 0xff; if (g_pfnGetCurrentProcessorNumberEx) g_pfnGetCurrentProcessorNumberEx(&ProcNum); else if (g_pfnGetCurrentProcessorNumber) { DWORD iCpu = g_pfnGetCurrentProcessorNumber(); Assert(iCpu < g_cRtMpWinMaxCpus); ProcNum.Number = iCpu; } else { #if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64) ProcNum.Number = ASMGetApicId(); #else # error "Not ported to this architecture." return NIL_RTCPUID; #endif } #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); #else return RTMpSetIndexFromCpuGroupMember(ProcNum.Group, ProcNum.Number); #endif }
RTDECL(bool) RTThreadPreemptIsEnabled(RTTHREAD hThread) { Assert(hThread == NIL_RTTHREAD); int32_t c = g_acPreemptDisabled[ASMGetApicId()]; AssertMsg(c >= 0 && c < 32, ("%d\n", c)); return c == 0 && ASMIntAreEnabled(); }
RTDECL(RTCPUID) RTMpCpuId(void) { #if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64) || defined(RT_ARCH_SPARC) || defined(RT_ARCH_SPARC64) return ASMGetApicId(); #else # error "Not ported to this architecture." return NIL_RTAPICID; #endif }
RTDECL(void) RTThreadPreemptDisable(PRTTHREADPREEMPTSTATE pState) { AssertPtr(pState); Assert(pState->u32Reserved == 0); /* No preemption on OS/2, so do our own accounting. */ int32_t c = ASMAtomicIncS32(&g_acPreemptDisabled[ASMGetApicId()]); AssertMsg(c > 0 && c < 32, ("%d\n", c)); pState->u32Reserved = c; RT_ASSERT_PREEMPT_CPUID_DISABLE(pState); }
RTDECL(void) RTThreadPreemptRestore(PRTTHREADPREEMPTSTATE pState) { AssertPtr(pState); AssertMsg(pState->u32Reserved > 0 && pState->u32Reserved < 32, ("%d\n", pState->u32Reserved)); RT_ASSERT_PREEMPT_CPUID_RESTORE(pState); /* No preemption on OS/2, so do our own accounting. */ int32_t volatile *pc = &g_acPreemptDisabled[ASMGetApicId()]; AssertMsg(pState->u32Reserved == (uint32_t)*pc, ("uchDummy=%d *pc=%d \n", pState->u32Reserved, *pc)); ASMAtomicUoWriteS32(pc, pState->u32Reserved - 1); pState->u32Reserved = 0; }
/** * Internal worker for getting the GIP CPU array index for the calling CPU. * * @returns Index into SUPGLOBALINFOPAGE::aCPUs or UINT16_MAX. * @param pGip The GIP. */ DECLINLINE(uint16_t) supGetGipCpuIndex(PSUPGLOBALINFOPAGE pGip) { uint16_t iGipCpu; #ifdef IN_RING3 if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS) { /* Storing the IDTR is normally very fast. */ uint16_t cbLim = ASMGetIdtrLimit(); uint16_t iCpuSet = cbLim - 256 * (ARCH_BITS == 64 ? 16 : 8); iCpuSet &= RTCPUSET_MAX_CPUS - 1; iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]; } else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS) { /* RDTSCP gives us what need need and more. */ uint32_t iCpuSet; ASMReadTscWithAux(&iCpuSet); iCpuSet &= RTCPUSET_MAX_CPUS - 1; iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]; } else { /* Get APIC ID via the slow CPUID instruction. */ uint8_t idApic = ASMGetApicId(); iGipCpu = pGip->aiCpuFromApicId[idApic]; } #elif defined(IN_RING0) /* Ring-0: Use use RTMpCpuId() (disables cli to avoid host OS assertions about unsafe CPU number usage). */ RTCCUINTREG uFlags = ASMIntDisableFlags(); int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId()); if (RT_LIKELY((unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx))) iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]; else iGipCpu = UINT16_MAX; ASMSetFlags(uFlags); # elif defined(IN_RC) /* Raw-mode context: We can get the host CPU set index via VMCPU. */ uint32_t iCpuSet = VMMGetCpu(&g_VM)->iHostCpuSet; if (RT_LIKELY(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx))) iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]; else iGipCpu = UINT16_MAX; #else # error "IN_RING3, IN_RC or IN_RING0 must be defined!" #endif return iGipCpu; }
/** * The slow case for SUPReadTsc where we need to apply deltas. * * Must only be called when deltas are applicable, so please do not call it * directly. * * @returns TSC with delta applied. * @param pGip Pointer to the GIP. * * @remarks May be called with interrupts disabled in ring-0! This is why the * ring-0 code doesn't attempt to figure the delta. * * @internal */ SUPDECL(uint64_t) SUPReadTscWithDelta(PSUPGLOBALINFOPAGE pGip) { uint64_t uTsc; uint16_t iGipCpu; AssertCompile(RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS)); AssertCompile(RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx) >= RTCPUSET_MAX_CPUS); Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO); /* * Read the TSC and get the corresponding aCPUs index. */ #ifdef IN_RING3 if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS) { /* RDTSCP gives us all we need, no loops/cli. */ uint32_t iCpuSet; uTsc = ASMReadTscWithAux(&iCpuSet); iCpuSet &= RTCPUSET_MAX_CPUS - 1; iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]; } else if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS) { /* Storing the IDTR is normally very quick, but we need to loop. */ uint32_t cTries = 0; for (;;) { uint16_t cbLim = ASMGetIdtrLimit(); uTsc = ASMReadTSC(); if (RT_LIKELY(ASMGetIdtrLimit() == cbLim)) { uint16_t iCpuSet = cbLim - 256 * (ARCH_BITS == 64 ? 16 : 8); iCpuSet &= RTCPUSET_MAX_CPUS - 1; iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]; break; } if (cTries >= 16) { iGipCpu = UINT16_MAX; break; } cTries++; } } else { /* Get APIC ID via the slow CPUID instruction, requires looping. */ uint32_t cTries = 0; for (;;) { uint8_t idApic = ASMGetApicId(); uTsc = ASMReadTSC(); if (RT_LIKELY(ASMGetApicId() == idApic)) { iGipCpu = pGip->aiCpuFromApicId[idApic]; break; } if (cTries >= 16) { iGipCpu = UINT16_MAX; break; } cTries++; } } #elif defined(IN_RING0) /* Ring-0: Use use RTMpCpuId(), no loops. */ RTCCUINTREG uFlags = ASMIntDisableFlags(); int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId()); if (RT_LIKELY((unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx))) iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]; else iGipCpu = UINT16_MAX; uTsc = ASMReadTSC(); ASMSetFlags(uFlags); # elif defined(IN_RC) /* Raw-mode context: We can get the host CPU set index via VMCPU, no loops. */ RTCCUINTREG uFlags = ASMIntDisableFlags(); /* Are already disable, but play safe. */ uint32_t iCpuSet = VMMGetCpu(&g_VM)->iHostCpuSet; if (RT_LIKELY(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx))) iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]; else iGipCpu = UINT16_MAX; uTsc = ASMReadTSC(); ASMSetFlags(uFlags); #else # error "IN_RING3, IN_RC or IN_RING0 must be defined!" #endif /* * If the delta is valid, apply it. */ if (RT_LIKELY(iGipCpu < pGip->cCpus)) { int64_t iTscDelta = pGip->aCPUs[iGipCpu].i64TSCDelta; if (RT_LIKELY(iTscDelta != INT64_MAX)) return uTsc - iTscDelta; # ifdef IN_RING3 /* * The delta needs calculating, call supdrv to get the TSC. */ int rc = SUPR3ReadTsc(&uTsc, NULL); if (RT_SUCCESS(rc)) return uTsc; AssertMsgFailed(("SUPR3ReadTsc -> %Rrc\n", rc)); uTsc = ASMReadTSC(); # endif /* IN_RING3 */ } /* * This shouldn't happen, especially not in ring-3 and raw-mode context. * But if it does, return something that's half useful. */ AssertMsgFailed(("iGipCpu=%d (%#x) cCpus=%d fGetGipCpu=%#x\n", iGipCpu, iGipCpu, pGip->cCpus, pGip->fGetGipCpu)); return uTsc; }