/**
* @interface_method_impl{PDMINETWORKUP,pfnAllocBuf}
*/
PDMBOTHCBDECL(int) drvNetShaperUp_AllocBuf(PPDMINETWORKUP pInterface, size_t cbMin,
PCPDMNETWORKGSO pGso, PPPDMSCATTERGATHER ppSgBuf)
{
PDRVNETSHAPER pThis = RT_FROM_MEMBER(pInterface, DRVNETSHAPER, CTX_SUFF(INetworkUp));
if (RT_UNLIKELY(!pThis->CTX_SUFF(pIBelowNet)))
return VERR_NET_DOWN;
//LogFlow(("drvNetShaperUp_AllocBuf: cb=%d\n", cbMin));
STAM_REL_COUNTER_ADD(&pThis->StatXmitBytesRequested, cbMin);
STAM_REL_COUNTER_INC(&pThis->StatXmitPktsRequested);
#ifdef IN_RING3
if (!PDMR3NsAllocateBandwidth(&pThis->Filter, cbMin))
{
STAM_REL_COUNTER_ADD(&pThis->StatXmitBytesDenied, cbMin);
STAM_REL_COUNTER_INC(&pThis->StatXmitPktsDenied);
return VERR_TRY_AGAIN;
}
#endif
#ifdef IN_RING0
if (!PDMR0NsAllocateBandwidth(&pThis->Filter, cbMin))
{
STAM_REL_COUNTER_ADD(&pThis->StatXmitBytesDenied, cbMin);
STAM_REL_COUNTER_INC(&pThis->StatXmitPktsDenied);
return VERR_TRY_AGAIN;
}
#endif
STAM_REL_COUNTER_ADD(&pThis->StatXmitBytesGranted, cbMin);
STAM_REL_COUNTER_INC(&pThis->StatXmitPktsGranted);
//LogFlow(("drvNetShaperUp_AllocBuf: got cb=%d\n", cbMin));
return pThis->CTX_SUFF(pIBelowNet)->pfnAllocBuf(pThis->CTX_SUFF(pIBelowNet), cbMin, pGso, ppSgBuf);
}
/**
* Common worker for the debug and normal APIs.
*
* @retval VINF_SUCCESS on success.
* @retval VERR_SEM_BUSY if the critsect was owned.
* @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
* @retval VERR_SEM_DESTROYED if RTCritSectDelete was called while waiting.
*
* @param pCritSect The critical section.
*/
static int pdmCritSectTryEnter(PPDMCRITSECT pCritSect, PCRTLOCKVALSRCPOS pSrcPos)
{
/*
* If the critical section has already been destroyed, then inform the caller.
*/
AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC,
("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic),
VERR_SEM_DESTROYED);
/*
* See if we're lucky.
*/
/* NOP ... */
if (pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP)
return VINF_SUCCESS;
RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pCritSect);
/* ... not owned ... */
if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
/* ... or nested. */
if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf)
{
ASMAtomicIncS32(&pCritSect->s.Core.cLockers);
ASMAtomicIncS32(&pCritSect->s.Core.cNestings);
Assert(pCritSect->s.Core.cNestings > 1);
return VINF_SUCCESS;
}
/* no spinning */
/*
* Return busy.
*/
#ifdef IN_RING3
STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionR3);
#else
STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock);
#endif
LogFlow(("PDMCritSectTryEnter: locked\n"));
return VERR_SEM_BUSY;
}
/**
* Allocate an item from a queue.
* The allocated item must be handed on to PDMR3QueueInsert() after the
* data have been filled in.
*
* @returns Pointer to allocated queue item.
* @returns NULL on failure. The queue is exhausted.
* @param pQueue The queue handle.
* @thread Any thread.
*/
VMMDECL(PPDMQUEUEITEMCORE) PDMQueueAlloc(PPDMQUEUE pQueue)
{
Assert(VALID_PTR(pQueue) && pQueue->CTX_SUFF(pVM));
PPDMQUEUEITEMCORE pNew;
uint32_t iNext;
uint32_t i;
do
{
i = pQueue->iFreeTail;
if (i == pQueue->iFreeHead)
{
STAM_REL_COUNTER_INC(&pQueue->StatAllocFailures);
return NULL;
}
pNew = pQueue->aFreeItems[i].CTX_SUFF(pItem);
iNext = (i + 1) % (pQueue->cItems + PDMQUEUE_FREE_SLACK);
} while (!ASMAtomicCmpXchgU32(&pQueue->iFreeTail, iNext, i));
return pNew;
}
/**
* Queue an item.
* The item must have been obtained using PDMQueueAlloc(). Once the item
* have been passed to this function it must not be touched!
*
* @param pQueue The queue handle.
* @param pItem The item to insert.
* @thread Any thread.
*/
VMMDECL(void) PDMQueueInsert(PPDMQUEUE pQueue, PPDMQUEUEITEMCORE pItem)
{
Assert(VALID_PTR(pQueue) && pQueue->CTX_SUFF(pVM));
Assert(VALID_PTR(pItem));
#if 0 /* the paranoid android version: */
void *pvNext;
do
{
pvNext = ASMAtomicUoReadPtr((void * volatile *)&pQueue->CTX_SUFF(pPending));
ASMAtomicUoWritePtr((void * volatile *)&pItem->CTX_SUFF(pNext), pvNext);
} while (!ASMAtomicCmpXchgPtr(&pQueue->CTX_SUFF(pPending), pItem, pvNext));
#else
PPDMQUEUEITEMCORE pNext;
do
{
pNext = pQueue->CTX_SUFF(pPending);
pItem->CTX_SUFF(pNext) = pNext;
} while (!ASMAtomicCmpXchgPtr(&pQueue->CTX_SUFF(pPending), pItem, pNext));
#endif
if (!pQueue->pTimer)
{
PVM pVM = pQueue->CTX_SUFF(pVM);
Log2(("PDMQueueInsert: VM_FF_PDM_QUEUES %d -> 1\n", VM_FF_ISSET(pVM, VM_FF_PDM_QUEUES)));
VM_FF_SET(pVM, VM_FF_PDM_QUEUES);
ASMAtomicBitSet(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT);
#ifdef IN_RING3
# ifdef VBOX_WITH_REM
REMR3NotifyQueuePending(pVM); /** @todo r=bird: we can remove REMR3NotifyQueuePending and let VMR3NotifyFF do the work. */
# endif
VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
#endif
}
STAM_REL_COUNTER_INC(&pQueue->StatInsert);
STAM_STATS({ ASMAtomicIncU32(&pQueue->cStatPending); });
/**
* Leaves a critical section entered with PDMCritSectEnter().
*
* @param pCritSect The PDM critical section to leave.
*/
VMMDECL(void) PDMCritSectLeave(PPDMCRITSECT pCritSect)
{
AssertMsg(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic));
Assert(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC);
/* Check for NOP sections before asserting ownership. */
if (pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP)
return;
/*
* Always check that the caller is the owner (screw performance).
*/
RTNATIVETHREAD const hNativeSelf = pdmCritSectGetNativeSelf(pCritSect);
AssertReleaseMsgReturnVoid(pCritSect->s.Core.NativeThreadOwner == hNativeSelf,
("%p %s: %p != %p; cLockers=%d cNestings=%d\n", pCritSect, R3STRING(pCritSect->s.pszName),
pCritSect->s.Core.NativeThreadOwner, hNativeSelf,
pCritSect->s.Core.cLockers, pCritSect->s.Core.cNestings));
Assert(pCritSect->s.Core.cNestings >= 1);
/*
* Nested leave.
*/
if (pCritSect->s.Core.cNestings > 1)
{
ASMAtomicDecS32(&pCritSect->s.Core.cNestings);
Assert(pCritSect->s.Core.cNestings >= 1);
ASMAtomicDecS32(&pCritSect->s.Core.cLockers);
Assert(pCritSect->s.Core.cLockers >= 0);
return;
}
#ifdef IN_RING0
# if 0 /** @todo Make SUPSemEventSignal interrupt safe (handle table++) and enable this for: defined(RT_OS_LINUX) || defined(RT_OS_OS2) */
if (1) /* SUPSemEventSignal is safe */
# else
if (ASMIntAreEnabled())
# endif
#endif
#if defined(IN_RING3) || defined(IN_RING0)
{
/*
* Leave for real.
*/
/* update members. */
# ifdef IN_RING3
RTSEMEVENT hEventToSignal = pCritSect->s.EventToSignal;
pCritSect->s.EventToSignal = NIL_RTSEMEVENT;
# if defined(PDMCRITSECT_STRICT)
if (pCritSect->s.Core.pValidatorRec->hThread != NIL_RTTHREAD)
RTLockValidatorRecExclReleaseOwnerUnchecked(pCritSect->s.Core.pValidatorRec);
# endif
Assert(!pCritSect->s.Core.pValidatorRec || pCritSect->s.Core.pValidatorRec->hThread == NIL_RTTHREAD);
# endif
ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD);
ASMAtomicDecS32(&pCritSect->s.Core.cNestings);
Assert(pCritSect->s.Core.cNestings == 0);
/* stop and decrement lockers. */
STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l);
ASMCompilerBarrier();
if (ASMAtomicDecS32(&pCritSect->s.Core.cLockers) >= 0)
{
/* Someone is waiting, wake up one of them. */
SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->s.Core.EventSem;
PSUPDRVSESSION pSession = pCritSect->s.CTX_SUFF(pVM)->pSession;
int rc = SUPSemEventSignal(pSession, hEvent);
AssertRC(rc);
}
# ifdef IN_RING3
/* Signal exit event. */
if (hEventToSignal != NIL_RTSEMEVENT)
{
LogBird(("Signalling %#x\n", hEventToSignal));
int rc = RTSemEventSignal(hEventToSignal);
AssertRC(rc);
}
# endif
# if defined(DEBUG_bird) && defined(IN_RING0)
VMMTrashVolatileXMMRegs();
# endif
}
#endif /* IN_RING3 || IN_RING0 */
#ifdef IN_RING0
else
#endif
#if defined(IN_RING0) || defined(IN_RC)
{
/*
* Try leave it.
*/
if (pCritSect->s.Core.cLockers == 0)
{
ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0);
RTNATIVETHREAD hNativeThread = pCritSect->s.Core.NativeThreadOwner;
ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l);
ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD);
if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, -1, 0))
return;
/* darn, someone raced in on us. */
ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeThread);
STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l);
Assert(pCritSect->s.Core.cNestings == 0);
ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1);
}
ASMAtomicOrU32(&pCritSect->s.Core.fFlags, PDMCRITSECT_FLAGS_PENDING_UNLOCK);
/*
* Queue the request.
*/
PVM pVM = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM);
PVMCPU pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
uint32_t i = pVCpu->pdm.s.cQueuedCritSectLeaves++;
LogFlow(("PDMCritSectLeave: [%d]=%p => R3\n", i, pCritSect));
AssertFatal(i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectsLeaves));
pVCpu->pdm.s.apQueuedCritSectsLeaves[i] = MMHyperCCToR3(pVM, pCritSect);
VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZUnlock);
}
#endif /* IN_RING0 || IN_RC */
}
/**
* Common worker for the debug and normal APIs.
*
* @returns VINF_SUCCESS if entered successfully.
* @returns rcBusy when encountering a busy critical section in GC/R0.
* @returns VERR_SEM_DESTROYED if the critical section is dead.
*
* @param pCritSect The PDM critical section to enter.
* @param rcBusy The status code to return when we're in GC or R0
* and the section is busy.
*/
DECL_FORCE_INLINE(int) pdmCritSectEnter(PPDMCRITSECT pCritSect, int rcBusy, PCRTLOCKVALSRCPOS pSrcPos)
{
Assert(pCritSect->s.Core.cNestings < 8); /* useful to catch incorrect locking */
Assert(pCritSect->s.Core.cNestings >= 0);
/*
* If the critical section has already been destroyed, then inform the caller.
*/
AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC,
("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic),
VERR_SEM_DESTROYED);
/*
* See if we're lucky.
*/
/* NOP ... */
if (pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP)
return VINF_SUCCESS;
RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pCritSect);
/* ... not owned ... */
if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
/* ... or nested. */
if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf)
{
ASMAtomicIncS32(&pCritSect->s.Core.cLockers);
ASMAtomicIncS32(&pCritSect->s.Core.cNestings);
Assert(pCritSect->s.Core.cNestings > 1);
return VINF_SUCCESS;
}
/*
* Spin for a bit without incrementing the counter.
*/
/** @todo Move this to cfgm variables since it doesn't make sense to spin on UNI
* cpu systems. */
int32_t cSpinsLeft = CTX_SUFF(PDMCRITSECT_SPIN_COUNT_);
while (cSpinsLeft-- > 0)
{
if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
ASMNopPause();
/** @todo Should use monitor/mwait on e.g. &cLockers here, possibly with a
cli'ed pendingpreemption check up front using sti w/ instruction fusing
for avoiding races. Hmm ... This is assuming the other party is actually
executing code on another CPU ... which we could keep track of if we
wanted. */
}
#ifdef IN_RING3
/*
* Take the slow path.
*/
return pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
#else
# ifdef IN_RING0
/** @todo If preemption is disabled it means we're in VT-x/AMD-V context
* and would be better off switching out of that while waiting for
* the lock. Several of the locks jumps back to ring-3 just to
* get the lock, the ring-3 code will then call the kernel to do
* the lock wait and when the call return it will call ring-0
* again and resume via in setjmp style. Not very efficient. */
# if 0
if (ASMIntAreEnabled()) /** @todo this can be handled as well by changing
* callers not prepared for longjmp/blocking to
* use PDMCritSectTryEnter. */
{
/*
* Leave HWACCM context while waiting if necessary.
*/
int rc;
if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
{
STAM_REL_COUNTER_ADD(&pCritSect->s.StatContentionRZLock, 1000000);
rc = pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
}
else
{
STAM_REL_COUNTER_ADD(&pCritSect->s.StatContentionRZLock, 1000000000);
PVM pVM = pCritSect->s.CTX_SUFF(pVM);
PVMCPU pVCpu = VMMGetCpu(pVM);
HWACCMR0Leave(pVM, pVCpu);
RTThreadPreemptRestore(NIL_RTTHREAD, ????);
rc = pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
RTThreadPreemptDisable(NIL_RTTHREAD, ????);
HWACCMR0Enter(pVM, pVCpu);
}
return rc;
}
# else
/*
* We preemption hasn't been disabled, we can block here in ring-0.
*/
if ( RTThreadPreemptIsEnabled(NIL_RTTHREAD)
&& ASMIntAreEnabled())
return pdmR3R0CritSectEnterContended(pCritSect, hNativeSelf, pSrcPos);
# endif
#endif /* IN_RING0 */
STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock);
/*
* Call ring-3 to acquire the critical section?
*/
if (rcBusy == VINF_SUCCESS)
{
PVM pVM = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM);
PVMCPU pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_ENTER, MMHyperCCToR3(pVM, pCritSect));
}
/*
* Return busy.
*/
LogFlow(("PDMCritSectEnter: locked => R3 (%Rrc)\n", rcBusy));
return rcBusy;
#endif /* !IN_RING3 */
}
/**
* @interface_method_impl{PDMINETWORKDOWN,pfnXmitPending}
*/
static DECLCALLBACK(void) drvR3NetShaperDown_XmitPending(PPDMINETWORKDOWN pInterface)
{
PDRVNETSHAPER pThis = RT_FROM_MEMBER(pInterface, DRVNETSHAPER, INetworkDown);
STAM_REL_COUNTER_INC(&pThis->StatXmitPendingCalled);
pThis->pIAboveNet->pfnXmitPending(pThis->pIAboveNet);
}
/**
* Process pending items in one queue.
*
* @returns Success indicator.
* If false the item the consumer said "enough!".
* @param pQueue The queue.
*/
static bool pdmR3QueueFlush(PPDMQUEUE pQueue)
{
STAM_PROFILE_START(&pQueue->StatFlushPrf,p);
/*
* Get the lists.
*/
PPDMQUEUEITEMCORE pItems = ASMAtomicXchgPtrT(&pQueue->pPendingR3, NULL, PPDMQUEUEITEMCORE);
RTRCPTR pItemsRC = ASMAtomicXchgRCPtr(&pQueue->pPendingRC, NIL_RTRCPTR);
RTR0PTR pItemsR0 = ASMAtomicXchgR0Ptr(&pQueue->pPendingR0, NIL_RTR0PTR);
AssertMsgReturn( pItemsR0
|| pItemsRC
|| pItems,
("Someone is racing us? This shouldn't happen!\n"),
true);
/*
* Reverse the list (it's inserted in LIFO order to avoid semaphores, remember).
*/
PPDMQUEUEITEMCORE pCur = pItems;
pItems = NULL;
while (pCur)
{
PPDMQUEUEITEMCORE pInsert = pCur;
pCur = pCur->pNextR3;
pInsert->pNextR3 = pItems;
pItems = pInsert;
}
/*
* Do the same for any pending RC items.
*/
while (pItemsRC)
{
PPDMQUEUEITEMCORE pInsert = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pQueue->pVMR3, pItemsRC);
pItemsRC = pInsert->pNextRC;
pInsert->pNextRC = NIL_RTRCPTR;
pInsert->pNextR3 = pItems;
pItems = pInsert;
}
/*
* Do the same for any pending R0 items.
*/
while (pItemsR0)
{
PPDMQUEUEITEMCORE pInsert = (PPDMQUEUEITEMCORE)MMHyperR0ToR3(pQueue->pVMR3, pItemsR0);
pItemsR0 = pInsert->pNextR0;
pInsert->pNextR0 = NIL_RTR0PTR;
pInsert->pNextR3 = pItems;
pItems = pInsert;
}
/*
* Feed the items to the consumer function.
*/
Log2(("pdmR3QueueFlush: pQueue=%p enmType=%d pItems=%p\n", pQueue, pQueue->enmType, pItems));
switch (pQueue->enmType)
{
case PDMQUEUETYPE_DEV:
while (pItems)
{
if (!pQueue->u.Dev.pfnCallback(pQueue->u.Dev.pDevIns, pItems))
break;
pCur = pItems;
pItems = pItems->pNextR3;
pdmR3QueueFreeItem(pQueue, pCur);
}
break;
case PDMQUEUETYPE_DRV:
while (pItems)
{
if (!pQueue->u.Drv.pfnCallback(pQueue->u.Drv.pDrvIns, pItems))
break;
pCur = pItems;
pItems = pItems->pNextR3;
pdmR3QueueFreeItem(pQueue, pCur);
}
break;
case PDMQUEUETYPE_INTERNAL:
while (pItems)
{
if (!pQueue->u.Int.pfnCallback(pQueue->pVMR3, pItems))
break;
pCur = pItems;
pItems = pItems->pNextR3;
pdmR3QueueFreeItem(pQueue, pCur);
}
break;
case PDMQUEUETYPE_EXTERNAL:
while (pItems)
{
if (!pQueue->u.Ext.pfnCallback(pQueue->u.Ext.pvUser, pItems))
break;
pCur = pItems;
pItems = pItems->pNextR3;
pdmR3QueueFreeItem(pQueue, pCur);
}
break;
default:
AssertMsgFailed(("Invalid queue type %d\n", pQueue->enmType));
break;
}
/*
* Success?
*/
if (pItems)
{
/*
* Reverse the list.
*/
pCur = pItems;
pItems = NULL;
while (pCur)
{
PPDMQUEUEITEMCORE pInsert = pCur;
pCur = pInsert->pNextR3;
pInsert->pNextR3 = pItems;
pItems = pInsert;
}
/*
* Insert the list at the tail of the pending list.
*/
for (;;)
{
if (ASMAtomicCmpXchgPtr(&pQueue->pPendingR3, pItems, NULL))
break;
PPDMQUEUEITEMCORE pPending = ASMAtomicXchgPtrT(&pQueue->pPendingR3, NULL, PPDMQUEUEITEMCORE);
if (pPending)
{
pCur = pPending;
while (pCur->pNextR3)
pCur = pCur->pNextR3;
pCur->pNextR3 = pItems;
pItems = pPending;
}
}
STAM_REL_COUNTER_INC(&pQueue->StatFlushLeftovers);
STAM_PROFILE_STOP(&pQueue->StatFlushPrf,p);
return false;
}
STAM_PROFILE_STOP(&pQueue->StatFlushPrf,p);
return true;
}
