void
MXUser_ReleaseRecLock(MXUserRecLock *lock) // IN/OUT:
{
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_REC);
ASSERT(Atomic_Read(&lock->refCount) > 0);
if (UNLIKELY(lock->vmmLock != NULL)) {
ASSERT(MXUserMX_UnlockRec);
(*MXUserMX_UnlockRec)(lock->vmmLock);
} else {
if (vmx86_stats) {
MXUserHeldStats *heldStats = Atomic_ReadPtr(&lock->heldStatsMem);
if (LIKELY(heldStats != NULL)) {
if (MXRecLockCount(&lock->recursiveLock) == 1) {
MXUserHeldStats *heldStats;
heldStats = Atomic_ReadPtr(&lock->heldStatsMem);
if (UNLIKELY(heldStats != NULL)) {
VmTimeType value;
MXUserHisto *histo = Atomic_ReadPtr(&heldStats->histo);
value = Hostinfo_SystemTimerNS() - heldStats->holdStart;
MXUserBasicStatsSample(&heldStats->data, value);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
}
}
}
}
if (vmx86_debug) {
if (MXRecLockCount(&lock->recursiveLock) == 0) {
MXUserDumpAndPanic(&lock->header,
"%s: Release of an unacquired recursive lock\n",
__FUNCTION__);
}
if (!MXRecLockIsOwner(&lock->recursiveLock)) {
MXUserDumpAndPanic(&lock->header,
"%s: Non-owner release of an recursive lock\n",
__FUNCTION__);
}
}
MXUserReleaseTracking(&lock->header);
MXRecLockRelease(&lock->recursiveLock);
}
}
Bool
MXUser_TryAcquireExclLock(MXUserExclLock *lock) // IN/OUT:
{
Bool success;
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_EXCL);
if (MXUserTryAcquireFail(lock->header.name)) {
return FALSE;
}
success = MXRecLockTryAcquire(&lock->recursiveLock);
if (success) {
MXUserAcquisitionTracking(&lock->header, FALSE);
if (vmx86_debug && (MXRecLockCount(&lock->recursiveLock) > 1)) {
MXUserDumpAndPanic(&lock->header,
"%s: Acquire on an acquired exclusive lock\n",
__FUNCTION__);
}
}
if (vmx86_stats) {
MXUserStats *stats = Atomic_ReadPtr(&lock->statsMem);
if (LIKELY(stats != NULL)) {
MXUserAcquisitionSample(&stats->acquisitionStats, success, !success,
0ULL);
}
}
return success;
}
static void
MXUserCondDestroyRecLock(MXUserRecLock *lock) // IN:
{
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_REC);
ASSERT(Atomic_Read(&lock->refCount) > 0);
if (Atomic_ReadDec32(&lock->refCount) == 1) {
if (lock->vmmLock == NULL) {
if (MXRecLockCount(&lock->recursiveLock) > 0) {
MXUserDumpAndPanic(&lock->header,
"%s: Destroy of an acquired recursive lock\n",
__FUNCTION__);
}
MXRecLockDestroy(&lock->recursiveLock);
MXUserRemoveFromList(&lock->header);
if (vmx86_stats) {
MXUserDisableStats(&lock->acquireStatsMem, &lock->heldStatsMem);
}
}
lock->header.signature = 0; // just in case...
free(lock->header.name);
lock->header.name = NULL;
free(lock);
}
}
static INLINE void
MXUserWaitInternal(MXRecLock *lock, // IN:
MXUserCondVar *condVar, // IN:
uint32 msecWait) // IN:
{
int err;
int lockCount = MXRecLockCount(lock);
/*
* When using the native lock found within the MXUser lock, be sure to
* decrement the count before the wait/sleep and increment it after the
* wait/sleep - the (native) wait/sleep will perform a lock release before
* the wait/sleep and a lock acquisition after the wait/sleep. The
* MXUser internal accounting information must be maintained.
*/
MXRecLockDecCount(lock, lockCount);
if (msecWait == MXUSER_WAIT_INFINITE) {
err = pthread_cond_wait(&condVar->condObject, &lock->nativeLock);
} else {
struct timeval curTime;
struct timespec endTime;
uint64 endNS;
/*
* pthread_cond_timedwait takes an absolute time. Yes, this is
* beyond ridiculous, and the justifications for this
* vs. relative time make no sense. But IIWII.
*/
#define A_BILLION (1000 * 1000 * 1000)
gettimeofday(&curTime, NULL);
endNS = ((uint64) curTime.tv_sec * A_BILLION) +
((uint64) curTime.tv_usec * 1000) +
((uint64) msecWait * (1000 * 1000));
endTime.tv_sec = (time_t) (endNS / A_BILLION);
endTime.tv_nsec = (long int) (endNS % A_BILLION);
#undef A_BILLION
err = pthread_cond_timedwait(&condVar->condObject, &lock->nativeLock,
&endTime);
}
MXRecLockIncCount(lock, lockCount);
if (err != 0) {
if (err != ETIMEDOUT) {
Panic("%s: failure %d on condVar (0x%p; %s)\n", __FUNCTION__, err,
condVar, condVar->header->name);
}
}
}
void
MXUser_AcquireRecLock(MXUserRecLock *lock) // IN/OUT:
{
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_REC);
ASSERT(Atomic_Read(&lock->refCount) > 0);
if (UNLIKELY(lock->vmmLock != NULL)) {
ASSERT(MXUserMX_LockRec);
(*MXUserMX_LockRec)(lock->vmmLock);
} else {
/* Rank checking is only done on the first acquisition */
MXUserAcquisitionTracking(&lock->header, TRUE);
if (vmx86_stats) {
VmTimeType value = 0;
MXUserAcquireStats *acquireStats;
acquireStats = Atomic_ReadPtr(&lock->acquireStatsMem);
MXRecLockAcquire(&lock->recursiveLock,
(acquireStats == NULL) ? NULL : &value);
if (LIKELY(acquireStats != NULL)) {
if (MXRecLockCount(&lock->recursiveLock) == 1) {
MXUserHeldStats *heldStats;
MXUserHisto *histo;
MXUserAcquisitionSample(&acquireStats->data, TRUE,
value > acquireStats->data.contentionDurationFloor,
value);
histo = Atomic_ReadPtr(&acquireStats->histo);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
heldStats = Atomic_ReadPtr(&lock->heldStatsMem);
if (UNLIKELY(heldStats != NULL)) {
heldStats->holdStart = Hostinfo_SystemTimerNS();
}
}
}
} else {
MXRecLockAcquire(&lock->recursiveLock,
NULL); // non-stats
}
}
}
static void
MXUserDumpExclLock(MXUserHeader *header) // IN:
{
MXUserExclLock *lock = (MXUserExclLock *) header;
Warning("%s: Exclusive lock @ %p\n", __FUNCTION__, lock);
Warning("\tsignature 0x%X\n", lock->header.signature);
Warning("\tname %s\n", lock->header.name);
Warning("\trank 0x%X\n", lock->header.rank);
Warning("\tserial number %u\n", lock->header.serialNumber);
Warning("\tcount %d\n", MXRecLockCount(&lock->recursiveLock));
Warning("\taddress of owner data %p\n",
&lock->recursiveLock.nativeThreadID);
}
void
MXUserDumpRWLock(MXUserHeader *header) // IN:
{
MXUserRWLock *lock = (MXUserRWLock *) header;
Warning("%s: Read-write lock @ 0x%p\n", __FUNCTION__, lock);
Warning("\tsignature 0x%X\n", lock->header.signature);
Warning("\tname %s\n", lock->header.name);
Warning("\trank 0x%X\n", lock->header.rank);
Warning("\tserial number %u\n", lock->header.serialNumber);
if (LIKELY(lock->useNative)) {
Warning("\taddress of native lock 0x%p\n", &lock->nativeLock);
} else {
Warning("\tcount %d\n", MXRecLockCount(&lock->recursiveLock));
}
Warning("\tholderCount %d\n", Atomic_Read(&lock->holderCount));
}
void
MXUser_AcquireExclLock(MXUserExclLock *lock) // IN/OUT:
{
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_EXCL);
MXUserAcquisitionTracking(&lock->header, TRUE);
if (vmx86_stats) {
VmTimeType value = 0;
MXUserStats *stats = Atomic_ReadPtr(&lock->statsMem);
MXRecLockAcquire(&lock->recursiveLock, (stats == NULL) ? NULL : &value);
if (LIKELY(stats != NULL)) {
MXUserHisto *histo;
MXUserAcquisitionSample(&stats->acquisitionStats, TRUE, value != 0,
value);
histo = Atomic_ReadPtr(&stats->acquisitionHisto);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
stats->holdStart = Hostinfo_SystemTimerNS();
}
} else {
MXRecLockAcquire(&lock->recursiveLock,
NULL); // non-stats
}
if (vmx86_debug && (MXRecLockCount(&lock->recursiveLock) > 1)) {
MXUserDumpAndPanic(&lock->header,
"%s: Acquire on an acquired exclusive lock\n",
__FUNCTION__);
}
}
static void
MXUserDumpRecLock(MXUserHeader *header) // IN:
{
MXUserRecLock *lock = (MXUserRecLock *) header;
Warning("%s: Recursive lock @ %p\n", __FUNCTION__, lock);
Warning("\tsignature 0x%X\n", lock->header.signature);
Warning("\tname %s\n", lock->header.name);
Warning("\trank 0x%X\n", lock->header.rank);
Warning("\tserial number %u\n", lock->header.bits.serialNumber);
Warning("\treference count %u\n", Atomic_Read(&lock->refCount));
if (lock->vmmLock == NULL) {
Warning("\tlock count %d\n", MXRecLockCount(&lock->recursiveLock));
Warning("\taddress of owner data %p\n",
&lock->recursiveLock.nativeThreadID);
} else {
Warning("\tvmmLock %p\n", lock->vmmLock);
}
}
void
MXUser_DestroyExclLock(MXUserExclLock *lock) // IN:
{
if (lock != NULL) {
MXUserValidateHeader(&lock->header, MXUSER_TYPE_EXCL);
if (MXRecLockCount(&lock->recursiveLock) > 0) {
MXUserDumpAndPanic(&lock->header,
"%s: Destroy of an acquired exclusive lock\n",
__FUNCTION__);
}
lock->header.signature = 0; // just in case...
MXRecLockDestroy(&lock->recursiveLock);
MXUserRemoveFromList(&lock->header);
if (vmx86_stats) {
MXUserStats *stats = Atomic_ReadPtr(&lock->statsMem);
if (LIKELY(stats != NULL)) {
MXUserAcquisitionStatsTearDown(&stats->acquisitionStats);
MXUserHistoTearDown(Atomic_ReadPtr(&stats->acquisitionHisto));
MXUserBasicStatsTearDown(&stats->heldStats);
MXUserHistoTearDown(Atomic_ReadPtr(&stats->heldHisto));
free(stats);
}
}
free(lock->header.name);
lock->header.name = NULL;
free(lock);
}
}
void
MXUser_ReleaseExclLock(MXUserExclLock *lock) // IN/OUT:
{
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_EXCL);
if (vmx86_stats) {
MXUserStats *stats = Atomic_ReadPtr(&lock->statsMem);
if (LIKELY(stats != NULL)) {
MXUserHisto *histo;
VmTimeType value = Hostinfo_SystemTimerNS() - stats->holdStart;
MXUserBasicStatsSample(&stats->heldStats, value);
histo = Atomic_ReadPtr(&stats->heldHisto);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
}
}
if (vmx86_debug && !MXRecLockIsOwner(&lock->recursiveLock)) {
int lockCount = MXRecLockCount(&lock->recursiveLock);
MXUserDumpAndPanic(&lock->header,
"%s: Non-owner release of an %s exclusive lock\n",
__FUNCTION__,
lockCount == 0 ? "unacquired" : "acquired");
}
MXUserReleaseTracking(&lock->header);
MXRecLockRelease(&lock->recursiveLock);
}
static INLINE void
MXUserWaitInternal(MXRecLock *lock, // IN:
MXUserCondVar *condVar, // IN:
uint32 msecWait) // IN:
{
int lockCount = MXRecLockCount(lock);
DWORD waitTime = (msecWait == MXUSER_WAIT_INFINITE) ? INFINITE : msecWait;
if (pSleepConditionVariableCS) {
/*
* When using the native lock found within the MXUser lock, be sure to
* decrement the count before the wait/sleep and increment it after the
* wait/sleep - the (native) wait/sleep will perform a lock release
* before the wait/sleep and a lock acquisition after the wait/sleep.
* The MXUser internal accounting information must be maintained.
*/
MXRecLockDecCount(lock, lockCount);
(*pSleepConditionVariableCS)(&condVar->x.condObject, &lock->nativeLock,
waitTime);
MXRecLockIncCount(lock, lockCount);
} else {
DWORD err;
Bool done = FALSE;
EnterCriticalSection(&condVar->x.compat.condVarLock);
condVar->x.compat.numWaiters++;
LeaveCriticalSection(&condVar->x.compat.condVarLock);
MXRecLockDecCount(lock, lockCount - 1);
MXRecLockRelease(lock);
do {
DWORD status = WaitForSingleObject(condVar->x.compat.signalEvent,
waitTime);
EnterCriticalSection(&condVar->x.compat.condVarLock);
ASSERT(condVar->x.compat.numWaiters > 0);
if (status == WAIT_OBJECT_0) {
if (condVar->x.compat.numForRelease > 0) {
condVar->x.compat.numWaiters--;
if (--condVar->x.compat.numForRelease == 0) {
ResetEvent(condVar->x.compat.signalEvent);
}
err = ERROR_SUCCESS;
done = TRUE;
}
} else {
condVar->x.compat.numWaiters--;
if (status == WAIT_TIMEOUT) {
if (msecWait == MXUSER_WAIT_INFINITE) {
err = ERROR_CALL_NOT_IMPLEMENTED; // ACK! "IMPOSSIBLE"
} else {
err = ERROR_SUCCESS;
}
} else if (status == WAIT_ABANDONED) {
err = ERROR_WAIT_NO_CHILDREN;
} else {
ASSERT(status == WAIT_FAILED);
err = GetLastError();
}
done = TRUE;
}
LeaveCriticalSection(&condVar->x.compat.condVarLock);
} while (!done);
MXRecLockAcquire(lock,
NULL); // non-stats
MXRecLockIncCount(lock, lockCount - 1);
if (err != ERROR_SUCCESS) {
Panic("%s: failure %d on condVar (0x%p; %s)\n", __FUNCTION__, err,
condVar, condVar->header->name);
}
}
}
