/** * Deals with the contended case in ring-3 and ring-0. * * @returns VINF_SUCCESS or VERR_SEM_DESTROYED. * @param pCritSect The critsect. * @param hNativeSelf The native thread handle. */ static int pdmR3R0CritSectEnterContended(PPDMCRITSECT pCritSect, RTNATIVETHREAD hNativeSelf, PCRTLOCKVALSRCPOS pSrcPos) { /* * Start waiting. */ if (ASMAtomicIncS32(&pCritSect->s.Core.cLockers) == 0) return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); # ifdef IN_RING3 STAM_COUNTER_INC(&pCritSect->s.StatContentionR3); # else STAM_COUNTER_INC(&pCritSect->s.StatContentionRZLock); # endif /* * The wait loop. */ PSUPDRVSESSION pSession = pCritSect->s.CTX_SUFF(pVM)->pSession; SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->s.Core.EventSem; # ifdef IN_RING3 # ifdef PDMCRITSECT_STRICT RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt(); int rc2 = RTLockValidatorRecExclCheckOrder(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); if (RT_FAILURE(rc2)) return rc2; # else RTTHREAD hThreadSelf = RTThreadSelf(); # endif # endif for (;;) { # ifdef PDMCRITSECT_STRICT int rc9 = RTLockValidatorRecExclCheckBlocking(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos, !(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NO_NESTING), RT_INDEFINITE_WAIT, RTTHREADSTATE_CRITSECT, true); if (RT_FAILURE(rc9)) return rc9; # elif defined(IN_RING3) RTThreadBlocking(hThreadSelf, RTTHREADSTATE_CRITSECT, true); # endif int rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_INDEFINITE_WAIT); # ifdef IN_RING3 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_CRITSECT); # endif if (RT_UNLIKELY(pCritSect->s.Core.u32Magic != RTCRITSECT_MAGIC)) return VERR_SEM_DESTROYED; if (rc == VINF_SUCCESS) return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); AssertMsg(rc == VERR_INTERRUPTED, ("rc=%Rrc\n", rc)); } /* won't get here */ }
/** * Common worker for the debug and normal APIs. * * @retval VINF_SUCCESS on success. * @retval VERR_SEM_BUSY if the critsect was owned. * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) * @retval VERR_SEM_DESTROYED if RTCritSectDelete was called while waiting. * * @param pCritSect The critical section. */ static int pdmCritSectTryEnter(PPDMCRITSECT pCritSect, PCRTLOCKVALSRCPOS pSrcPos) { /* * If the critical section has already been destroyed, then inform the caller. */ AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic), VERR_SEM_DESTROYED); /* * See if we're lucky. */ /* NOP ... */ if (pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP) return VINF_SUCCESS; RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pCritSect); /* ... not owned ... */ if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1)) return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); /* ... or nested. */ if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf) { ASMAtomicIncS32(&pCritSect->s.Core.cLockers); ASMAtomicIncS32(&pCritSect->s.Core.cNestings); Assert(pCritSect->s.Core.cNestings > 1); return VINF_SUCCESS; } /* no spinning */ /* * Return busy. */ #ifdef IN_RING3 STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionR3); #else STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock); #endif LogFlow(("PDMCritSectTryEnter: locked\n")); return VERR_SEM_BUSY; }
/** * Common worker for the debug and normal APIs. * * @returns VINF_SUCCESS if entered successfully. * @returns rcBusy when encountering a busy critical section in GC/R0. * @returns VERR_SEM_DESTROYED if the critical section is dead. * * @param pCritSect The PDM critical section to enter. * @param rcBusy The status code to return when we're in GC or R0 * and the section is busy. */ DECL_FORCE_INLINE(int) pdmCritSectEnter(PPDMCRITSECT pCritSect, int rcBusy, PCRTLOCKVALSRCPOS pSrcPos) { Assert(pCritSect->s.Core.cNestings < 8); /* useful to catch incorrect locking */ Assert(pCritSect->s.Core.cNestings >= 0); /* * If the critical section has already been destroyed, then inform the caller. */ AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic), VERR_SEM_DESTROYED); /* * See if we're lucky. */ /* NOP ... */ if (pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP) return VINF_SUCCESS; RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pCritSect); /* ... not owned ... */ if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1)) return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); /* ... or nested. */ if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf) { ASMAtomicIncS32(&pCritSect->s.Core.cLockers); ASMAtomicIncS32(&pCritSect->s.Core.cNestings); Assert(pCritSect->s.Core.cNestings > 1); return VINF_SUCCESS; } /* * Spin for a bit without incrementing the counter. */ /** @todo Move this to cfgm variables since it doesn't make sense to spin on UNI * cpu systems. */ int32_t cSpinsLeft = CTX_SUFF(PDMCRITSECT_SPIN_COUNT_); while (cSpinsLeft-- > 0) { if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1)) return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); ASMNopPause(); /** @todo Should use monitor/mwait on e.g. &cLockers here, possibly with a cli'ed pendingpreemption check up front using sti w/ instruction fusing for avoiding races. Hmm ... This is assuming the other party is actually executing code on another CPU ... which we could keep track of if we wanted. */ } #ifdef IN_RING3 /* * Take the slow path. */ return pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos); #else # ifdef IN_RING0 /** @todo If preemption is disabled it means we're in VT-x/AMD-V context * and would be better off switching out of that while waiting for * the lock. Several of the locks jumps back to ring-3 just to * get the lock, the ring-3 code will then call the kernel to do * the lock wait and when the call return it will call ring-0 * again and resume via in setjmp style. Not very efficient. */ # if 0 if (ASMIntAreEnabled()) /** @todo this can be handled as well by changing * callers not prepared for longjmp/blocking to * use PDMCritSectTryEnter. */ { /* * Leave HWACCM context while waiting if necessary. */ int rc; if (RTThreadPreemptIsEnabled(NIL_RTTHREAD)) { STAM_REL_COUNTER_ADD(&pCritSect->s.StatContentionRZLock, 1000000); rc = pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos); } else { STAM_REL_COUNTER_ADD(&pCritSect->s.StatContentionRZLock, 1000000000); PVM pVM = pCritSect->s.CTX_SUFF(pVM); PVMCPU pVCpu = VMMGetCpu(pVM); HWACCMR0Leave(pVM, pVCpu); RTThreadPreemptRestore(NIL_RTTHREAD, ????); rc = pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos); RTThreadPreemptDisable(NIL_RTTHREAD, ????); HWACCMR0Enter(pVM, pVCpu); } return rc; } # else /* * We preemption hasn't been disabled, we can block here in ring-0. */ if ( RTThreadPreemptIsEnabled(NIL_RTTHREAD) && ASMIntAreEnabled()) return pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos); # endif #endif /* IN_RING0 */ STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock); /* * Call ring-3 to acquire the critical section? */ if (rcBusy == VINF_SUCCESS) { PVM pVM = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM); PVMCPU pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_ENTER, MMHyperCCToR3(pVM, pCritSect)); } /* * Return busy. */ LogFlow(("PDMCritSectEnter: locked => R3 (%Rrc)\n", rcBusy)); return rcBusy; #endif /* !IN_RING3 */ }
/** * Deals with the contended case in ring-3 and ring-0. * * @retval VINF_SUCCESS on success. * @retval VERR_SEM_DESTROYED if destroyed. * * @param pCritSect The critsect. * @param hNativeSelf The native thread handle. */ static int pdmR3R0CritSectEnterContended(PPDMCRITSECT pCritSect, RTNATIVETHREAD hNativeSelf, PCRTLOCKVALSRCPOS pSrcPos) { /* * Start waiting. */ if (ASMAtomicIncS32(&pCritSect->s.Core.cLockers) == 0) return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); # ifdef IN_RING3 STAM_COUNTER_INC(&pCritSect->s.StatContentionR3); # else STAM_COUNTER_INC(&pCritSect->s.StatContentionRZLock); # endif /* * The wait loop. */ PSUPDRVSESSION pSession = pCritSect->s.CTX_SUFF(pVM)->pSession; SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->s.Core.EventSem; # ifdef IN_RING3 # ifdef PDMCRITSECT_STRICT RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt(); int rc2 = RTLockValidatorRecExclCheckOrder(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); if (RT_FAILURE(rc2)) return rc2; # else RTTHREAD hThreadSelf = RTThreadSelf(); # endif # endif for (;;) { /* * Do the wait. * * In ring-3 this gets cluttered by lock validation and thread state * maintainence. * * In ring-0 we have to deal with the possibility that the thread has * been signalled and the interruptible wait function returning * immediately. In that case we do normal R0/RC rcBusy handling. */ # ifdef IN_RING3 # ifdef PDMCRITSECT_STRICT int rc9 = RTLockValidatorRecExclCheckBlocking(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos, !(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NO_NESTING), RT_INDEFINITE_WAIT, RTTHREADSTATE_CRITSECT, true); if (RT_FAILURE(rc9)) return rc9; # else RTThreadBlocking(hThreadSelf, RTTHREADSTATE_CRITSECT, true); # endif int rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_INDEFINITE_WAIT); RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_CRITSECT); # else /* IN_RING0 */ int rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_INDEFINITE_WAIT); # endif /* IN_RING0 */ /* * Deal with the return code and critsect destruction. */ if (RT_UNLIKELY(pCritSect->s.Core.u32Magic != RTCRITSECT_MAGIC)) return VERR_SEM_DESTROYED; if (rc == VINF_SUCCESS) return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); AssertMsg(rc == VERR_INTERRUPTED, ("rc=%Rrc\n", rc)); # ifdef IN_RING0 /* Something is pending (signal, APC, debugger, whatever), just go back to ring-3 so the kernel can deal with it when leaving kernel context. Note! We've incremented cLockers already and cannot safely decrement it without creating a race with PDMCritSectLeave, resulting in spurious wakeups. */ PVM pVM = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM); PVMCPU pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); rc = VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_VM_R0_PREEMPT, NULL); AssertRC(rc); # endif } /* won't get here */ }