/* static */ void
ES_NativeStackFrame::InitializeFrame(void *frame_start, ES_FrameStackIterator frame, void **&next_frame, BOOL include_return_address)
{
ES_Code *code = frame.GetCode();
ES_CodeWord *codeword = frame.GetCodeWord();
void **ptr = reinterpret_cast<void **>(frame_start);
if (include_return_address)
*(ptr - GetFrameSize(code, TRUE) / sizeof(void *)) = GetReturnAddress(code, codeword);
*next_frame = --ptr;
next_frame = ptr;
*--ptr = frame.GetRegisterFrame();
*--ptr = code;
*--ptr = frame.GetArgumentsObject();
#ifdef ARCHITECTURE_MIPS
*--ptr = reinterpret_cast<void *>(static_cast<UINTPTR>(frame.GetArgumentsCount()));
#endif
if (code->CanHaveVariableObject())
*--ptr = frame.GetVariableObject();
#ifndef ARCHITECTURE_MIPS
*--ptr = reinterpret_cast<void *>(static_cast<UINTPTR>(frame.GetArgumentsCount()));
#endif
}
void
MXUser_DownSemaphore(MXUserSemaphore *sema) // IN/OUT:
{
int err;
ASSERT(sema);
MXUserValidateHeader(&sema->header, MXUSER_TYPE_SEMA);
Atomic_Inc(&sema->activeUserCount);
MXUserAcquisitionTracking(&sema->header, TRUE); // rank checking
if (vmx86_stats) {
VmTimeType start = 0;
Bool tryDownSuccess = FALSE;
MXUserAcquireStats *acquireStats;
acquireStats = Atomic_ReadPtr(&sema->acquireStatsMem);
if (LIKELY(acquireStats != NULL)) {
start = Hostinfo_SystemTimerNS();
}
err = MXUserTryDown(&sema->nativeSemaphore, &tryDownSuccess);
if (LIKELY(err == 0)) {
if (!tryDownSuccess) {
err = MXUserDown(&sema->nativeSemaphore);
}
}
if (LIKELY((err == 0) && (acquireStats != NULL))) {
MXUserHisto *histo;
VmTimeType value = Hostinfo_SystemTimerNS() - start;
MXUserAcquisitionSample(&acquireStats->data, TRUE,
!tryDownSuccess, value);
histo = Atomic_ReadPtr(&acquireStats->histo);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
}
} else {
err = MXUserDown(&sema->nativeSemaphore);
}
if (UNLIKELY(err != 0)) {
MXUserDumpAndPanic(&sema->header, "%s: Internal error (%d)\n",
__FUNCTION__, err);
}
MXUserReleaseTracking(&sema->header);
Atomic_Dec(&sema->activeUserCount);
}
MXUserRecLock *
MXUser_CreateRecLock(const char *userName, // IN:
MX_Rank rank) // IN:
{
uint32 statsMode;
char *properName;
MXUserRecLock *lock = Util_SafeCalloc(1, sizeof *lock);
if (userName == NULL) {
properName = Str_SafeAsprintf(NULL, "R-%p", GetReturnAddress());
} else {
properName = Util_SafeStrdup(userName);
}
if (!MXRecLockInit(&lock->recursiveLock)) {
free(properName);
free(lock);
return NULL;
}
lock->vmmLock = NULL;
Atomic_Write(&lock->refCount, 1);
lock->header.signature = MXUserGetSignature(MXUSER_TYPE_REC);
lock->header.name = properName;
lock->header.rank = rank;
lock->header.bits.serialNumber = MXUserAllocSerialNumber();
lock->header.dumpFunc = MXUserDumpRecLock;
statsMode = MXUserStatsMode();
switch (statsMode) {
case 0:
MXUserDisableStats(&lock->acquireStatsMem, &lock->heldStatsMem);
lock->header.statsFunc = NULL;
break;
case 1:
MXUserEnableStats(&lock->acquireStatsMem, NULL);
lock->header.statsFunc = MXUserStatsActionRec;
break;
case 2:
MXUserEnableStats(&lock->acquireStatsMem, &lock->heldStatsMem);
lock->header.statsFunc = MXUserStatsActionRec;
break;
default:
Panic("%s: unknown stats mode: %d!\n", __FUNCTION__, statsMode);
}
MXUserAddToList(&lock->header);
return lock;
}
Bool IsCallerMainModule(LPVOID lpAddress)
{
DWORD dwAddress = GetReturnAddress(lpAddress);
for (DWORD i = 0; i != dwImageSections; ++i)
{
if (dwAddress >= ImageSections[i].dwAddressStart && dwAddress < ImageSections[i].dwAddressEnd)
return True;
}
return False;
}
void
MXUser_ReleaseRecLock(MXUserRecLock *lock) // IN/OUT:
{
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_REC);
ASSERT(Atomic_Read(&lock->refCount) > 0);
if (UNLIKELY(lock->vmmLock != NULL)) {
ASSERT(MXUserMX_UnlockRec);
(*MXUserMX_UnlockRec)(lock->vmmLock);
} else {
if (vmx86_stats) {
MXUserHeldStats *heldStats = Atomic_ReadPtr(&lock->heldStatsMem);
if (LIKELY(heldStats != NULL)) {
if (MXRecLockCount(&lock->recursiveLock) == 1) {
MXUserHeldStats *heldStats;
heldStats = Atomic_ReadPtr(&lock->heldStatsMem);
if (UNLIKELY(heldStats != NULL)) {
VmTimeType value;
MXUserHisto *histo = Atomic_ReadPtr(&heldStats->histo);
value = Hostinfo_SystemTimerNS() - heldStats->holdStart;
MXUserBasicStatsSample(&heldStats->data, value);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
}
}
}
}
if (vmx86_debug) {
if (MXRecLockCount(&lock->recursiveLock) == 0) {
MXUserDumpAndPanic(&lock->header,
"%s: Release of an unacquired recursive lock\n",
__FUNCTION__);
}
if (!MXRecLockIsOwner(&lock->recursiveLock)) {
MXUserDumpAndPanic(&lock->header,
"%s: Non-owner release of an recursive lock\n",
__FUNCTION__);
}
}
MXUserReleaseTracking(&lock->header);
MXRecLockRelease(&lock->recursiveLock);
}
}
void
MXUser_AcquireRecLock(MXUserRecLock *lock) // IN/OUT:
{
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_REC);
ASSERT(Atomic_Read(&lock->refCount) > 0);
if (UNLIKELY(lock->vmmLock != NULL)) {
ASSERT(MXUserMX_LockRec);
(*MXUserMX_LockRec)(lock->vmmLock);
} else {
/* Rank checking is only done on the first acquisition */
MXUserAcquisitionTracking(&lock->header, TRUE);
if (vmx86_stats) {
VmTimeType value = 0;
MXUserAcquireStats *acquireStats;
acquireStats = Atomic_ReadPtr(&lock->acquireStatsMem);
MXRecLockAcquire(&lock->recursiveLock,
(acquireStats == NULL) ? NULL : &value);
if (LIKELY(acquireStats != NULL)) {
if (MXRecLockCount(&lock->recursiveLock) == 1) {
MXUserHeldStats *heldStats;
MXUserHisto *histo;
MXUserAcquisitionSample(&acquireStats->data, TRUE,
value > acquireStats->data.contentionDurationFloor,
value);
histo = Atomic_ReadPtr(&acquireStats->histo);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
heldStats = Atomic_ReadPtr(&lock->heldStatsMem);
if (UNLIKELY(heldStats != NULL)) {
heldStats->holdStart = Hostinfo_SystemTimerNS();
}
}
}
} else {
MXRecLockAcquire(&lock->recursiveLock,
NULL); // non-stats
}
}
}
MXUserExclLock *
MXUser_CreateExclLock(const char *userName, // IN:
MX_Rank rank) // IN:
{
Bool doStats;
char *properName;
MXUserExclLock *lock;
lock = Util_SafeCalloc(1, sizeof(*lock));
if (userName == NULL) {
properName = Str_SafeAsprintf(NULL, "X-%p", GetReturnAddress());
} else {
properName = Util_SafeStrdup(userName);
}
if (!MXRecLockInit(&lock->recursiveLock)) {
free(properName);
free(lock);
return NULL;
}
lock->header.signature = MXUserGetSignature(MXUSER_TYPE_EXCL);
lock->header.name = properName;
lock->header.rank = rank;
lock->header.serialNumber = MXUserAllocSerialNumber();
lock->header.dumpFunc = MXUserDumpExclLock;
if (vmx86_stats) {
doStats = MXUserStatsEnabled();
} else {
doStats = FALSE;
}
if (doStats) {
MXUser_ControlExclLock(lock, MXUSER_CONTROL_ENABLE_STATS);
} else {
lock->header.statsFunc = NULL;
Atomic_WritePtr(&lock->statsMem, NULL);
}
MXUserAddToList(&lock->header);
return lock;
}
void TransitionFrame::UpdateRegDisplay(const PREGDISPLAY pRD)
{
LIMITED_METHOD_CONTRACT;
pRD->IsCallerContextValid = FALSE;
pRD->IsCallerSPValid = FALSE; // Don't add usage of this field. This is only temporary.
pRD->pCurrentContext->Rip = GetReturnAddress();
pRD->pCurrentContext->Rsp = GetSP();
UpdateRegDisplayFromCalleeSavedRegisters(pRD, GetCalleeSavedRegisters());
ClearRegDisplayArgumentAndScratchRegisters(pRD);
SyncRegDisplayToCurrentContext(pRD);
LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK TransitionFrame::UpdateRegDisplay(rip:%p, rsp:%p)\n", pRD->ControlPC, pRD->SP));
}
char *
Str_Strcpy(char *buf, // OUT
const char *src, // IN
size_t maxSize) // IN
{
size_t len;
len = strlen(src);
if (len >= maxSize) {
#ifdef GetReturnAddress
Panic("%s:%d Buffer too small 0x%p\n", __FILE__, __LINE__, GetReturnAddress());
#else
Panic("%s:%d Buffer too small\n", __FILE__, __LINE__);
#endif
}
return memcpy(buf, src, len + 1);
}
MXUserSemaphore *
MXUser_CreateSemaphore(const char *userName, // IN:
MX_Rank rank) // IN:
{
char *properName;
MXUserSemaphore *sema;
sema = Util_SafeCalloc(1, sizeof(*sema));
if (userName == NULL) {
properName = Str_SafeAsprintf(NULL, "Sema-%p", GetReturnAddress());
} else {
properName = Util_SafeStrdup(userName);
}
if (LIKELY(MXUserInit(&sema->nativeSemaphore) == 0)) {
sema->header.signature = MXUserGetSignature(MXUSER_TYPE_SEMA);
sema->header.name = properName;
sema->header.rank = rank;
sema->header.serialNumber = MXUserAllocSerialNumber();
sema->header.dumpFunc = MXUserDumpSemaphore;
if (MXUserStatsMode() == 0) {
sema->header.statsFunc = NULL;
Atomic_WritePtr(&sema->acquireStatsMem, NULL);
} else {
MXUserAcquireStats *acquireStats;
acquireStats = Util_SafeCalloc(1, sizeof(*acquireStats));
MXUserAcquisitionStatsSetUp(&acquireStats->data);
sema->header.statsFunc = MXUserStatsActionSema;
Atomic_WritePtr(&sema->acquireStatsMem, acquireStats);
}
MXUserAddToList(&sema->header);
} else {
free(properName);
free(sema);
sema = NULL;
}
return sema;
}
void
MXUser_AcquireExclLock(MXUserExclLock *lock) // IN/OUT:
{
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_EXCL);
MXUserAcquisitionTracking(&lock->header, TRUE);
if (vmx86_stats) {
VmTimeType value = 0;
MXUserStats *stats = Atomic_ReadPtr(&lock->statsMem);
MXRecLockAcquire(&lock->recursiveLock, (stats == NULL) ? NULL : &value);
if (LIKELY(stats != NULL)) {
MXUserHisto *histo;
MXUserAcquisitionSample(&stats->acquisitionStats, TRUE, value != 0,
value);
histo = Atomic_ReadPtr(&stats->acquisitionHisto);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
stats->holdStart = Hostinfo_SystemTimerNS();
}
} else {
MXRecLockAcquire(&lock->recursiveLock,
NULL); // non-stats
}
if (vmx86_debug && (MXRecLockCount(&lock->recursiveLock) > 1)) {
MXUserDumpAndPanic(&lock->header,
"%s: Acquire on an acquired exclusive lock\n",
__FUNCTION__);
}
}
void
MXUser_ReleaseExclLock(MXUserExclLock *lock) // IN/OUT:
{
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_EXCL);
if (vmx86_stats) {
MXUserStats *stats = Atomic_ReadPtr(&lock->statsMem);
if (LIKELY(stats != NULL)) {
MXUserHisto *histo;
VmTimeType value = Hostinfo_SystemTimerNS() - stats->holdStart;
MXUserBasicStatsSample(&stats->heldStats, value);
histo = Atomic_ReadPtr(&stats->heldHisto);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
}
}
if (vmx86_debug && !MXRecLockIsOwner(&lock->recursiveLock)) {
int lockCount = MXRecLockCount(&lock->recursiveLock);
MXUserDumpAndPanic(&lock->header,
"%s: Non-owner release of an %s exclusive lock\n",
__FUNCTION__,
lockCount == 0 ? "unacquired" : "acquired");
}
MXUserReleaseTracking(&lock->header);
MXRecLockRelease(&lock->recursiveLock);
}
void
MXUser_ReleaseRWLock(MXUserRWLock *lock) // IN/OUT:
{
HolderContext *myContext;
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_RW);
myContext = MXUserGetHolderContext(lock);
if (vmx86_stats) {
MXUserStats *stats = Atomic_ReadPtr(&lock->statsMem);
if (LIKELY(stats != NULL)) {
MXUserHisto *histo;
VmTimeType duration = Hostinfo_SystemTimerNS() - myContext->holdStart;
/*
* The statistics are not always atomically safe so protect them
* when necessary
*/
if ((myContext->state == RW_LOCKED_FOR_READ) && lock->useNative) {
MXRecLockAcquire(&lock->recursiveLock,
NULL); // non-stats
}
MXUserBasicStatsSample(&stats->heldStats, duration);
histo = Atomic_ReadPtr(&stats->heldHisto);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, duration, GetReturnAddress());
}
if ((myContext->state == RW_LOCKED_FOR_READ) && lock->useNative) {
MXRecLockRelease(&lock->recursiveLock);
}
}
}
if (UNLIKELY(myContext->state == RW_UNLOCKED)) {
uint32 lockCount = Atomic_Read(&lock->holderCount);
MXUserDumpAndPanic(&lock->header,
"%s: Non-owner release of an %s read-write lock\n",
__FUNCTION__,
lockCount == 0 ? "unacquired" : "acquired");
}
MXUserReleaseTracking(&lock->header);
Atomic_Dec(&lock->holderCount);
if (LIKELY(lock->useNative)) {
int err = MXUserNativeRWRelease(&lock->nativeLock,
myContext->state == RW_LOCKED_FOR_READ);
if (UNLIKELY(err != 0)) {
MXUserDumpAndPanic(&lock->header, "%s: Internal error (%d)\n",
__FUNCTION__, err);
}
} else {
ASSERT(Atomic_Read(&lock->holderCount) == 0);
MXRecLockRelease(&lock->recursiveLock);
}
myContext->state = RW_UNLOCKED;
}
static INLINE void
MXUserAcquisition(MXUserRWLock *lock, // IN/OUT:
Bool forRead) // IN:
{
HolderContext *myContext;
ASSERT(lock);
MXUserValidateHeader(&lock->header, MXUSER_TYPE_RW);
MXUserAcquisitionTracking(&lock->header, TRUE);
myContext = MXUserGetHolderContext(lock);
if (UNLIKELY(myContext->state != RW_UNLOCKED)) {
MXUserDumpAndPanic(&lock->header,
"%s: AcquireFor%s after AcquireFor%s\n",
__FUNCTION__,
forRead ? "Read" : "Write",
(myContext->state == RW_LOCKED_FOR_READ) ? "Read" :
"Write");
}
if (vmx86_stats) {
VmTimeType value;
MXUserStats *stats = Atomic_ReadPtr(&lock->statsMem);
if (lock->useNative) {
int err = 0;
Bool contended;
VmTimeType begin = Hostinfo_SystemTimerNS();
contended = MXUserNativeRWAcquire(&lock->nativeLock, forRead, &err);
value = contended ? Hostinfo_SystemTimerNS() - begin : 0;
if (UNLIKELY(err != 0)) {
MXUserDumpAndPanic(&lock->header, "%s: Error %d: contended %d\n",
__FUNCTION__, err, contended);
}
} else {
value = 0;
MXRecLockAcquire(&lock->recursiveLock,
(stats == NULL) ? NULL : &value);
}
if (LIKELY(stats != NULL)) {
MXUserHisto *histo;
/*
* The statistics are not atomically safe so protect them when
* necessary.
*/
if (forRead && lock->useNative) {
MXRecLockAcquire(&lock->recursiveLock,
NULL); // non-stats
}
MXUserAcquisitionSample(&stats->acquisitionStats, TRUE, value != 0,
value);
histo = Atomic_ReadPtr(&stats->acquisitionHisto);
if (UNLIKELY(histo != NULL)) {
MXUserHistoSample(histo, value, GetReturnAddress());
}
if (forRead && lock->useNative) {
MXRecLockRelease(&lock->recursiveLock);
}
myContext->holdStart = Hostinfo_SystemTimerNS();
}
} else {
if (LIKELY(lock->useNative)) {
int err = 0;
MXUserNativeRWAcquire(&lock->nativeLock, forRead, &err);
if (UNLIKELY(err != 0)) {
MXUserDumpAndPanic(&lock->header, "%s: Error %d\n",
__FUNCTION__, err);
}
} else {
MXRecLockAcquire(&lock->recursiveLock,
NULL); // non-stats
}
}
if (!forRead || !lock->useNative) {
ASSERT(Atomic_Read(&lock->holderCount) == 0);
}
Atomic_Inc(&lock->holderCount);
myContext->state = forRead ? RW_LOCKED_FOR_READ : RW_LOCKED_FOR_WRITE;
}
MXUserRWLock *
MXUser_CreateRWLock(const char *userName, // IN:
MX_Rank rank) // IN:
{
Bool lockInited;
char *properName;
MXUserRWLock *lock;
Bool useNative = MXUserNativeRWSupported();
lock = Util_SafeCalloc(1, sizeof(*lock));
if (userName == NULL) {
if (LIKELY(useNative)) {
properName = Str_SafeAsprintf(NULL, "RW-%p", GetReturnAddress());
} else {
/* emulated */
properName = Str_SafeAsprintf(NULL, "RWemul-%p", GetReturnAddress());
}
} else {
properName = Util_SafeStrdup(userName);
}
lock->header.signature = MXUserGetSignature(MXUSER_TYPE_RW);
lock->header.name = properName;
lock->header.rank = rank;
lock->header.serialNumber = MXUserAllocSerialNumber();
lock->header.dumpFunc = MXUserDumpRWLock;
/*
* Always attempt to use native locks when they are available. If, for some
* reason, a native lock should be available but isn't, fall back to using
* an internal recursive lock - something is better than nothing.
*/
lock->useNative = useNative && MXUserNativeRWInit(&lock->nativeLock);
lockInited = MXRecLockInit(&lock->recursiveLock);
if (LIKELY(lockInited)) {
Bool doStats;
lock->holderTable = HashTable_Alloc(256,
HASH_INT_KEY | HASH_FLAG_ATOMIC,
MXUserFreeHashEntry);
if (vmx86_stats) {
doStats = MXUserStatsEnabled();
} else {
doStats = FALSE;
}
if (doStats) {
MXUser_ControlRWLock(lock, MXUSER_CONTROL_ENABLE_STATS);
} else {
lock->header.statsFunc = NULL;
Atomic_WritePtr(&lock->statsMem, NULL);
}
MXUserAddToList(&lock->header);
} else {
if (lock->useNative) {
MXUserNativeRWDestroy(&lock->nativeLock);
}
free(properName);
free(lock);
lock = NULL;
}
return lock;
}
MXUserRWLock *
MXUser_CreateRWLock(const char *userName, // IN:
MX_Rank rank) // IN:
{
Bool lockInited;
char *properName;
Bool useNative = MXUserNativeRWSupported();
MXUserRWLock *lock = Util_SafeCalloc(1, sizeof *lock);
if (userName == NULL) {
if (LIKELY(useNative)) {
properName = Str_SafeAsprintf(NULL, "RW-%p", GetReturnAddress());
} else {
/* emulated */
properName = Str_SafeAsprintf(NULL, "RWemul-%p", GetReturnAddress());
}
} else {
properName = Util_SafeStrdup(userName);
}
lock->header.signature = MXUserGetSignature(MXUSER_TYPE_RW);
lock->header.name = properName;
lock->header.rank = rank;
lock->header.bits.serialNumber = MXUserAllocSerialNumber();
lock->header.dumpFunc = MXUserDumpRWLock;
/*
* Always attempt to use native locks when they are available. If, for some
* reason, a native lock should be available but isn't, fall back to using
* an internal recursive lock - something is better than nothing.
*/
lock->useNative = useNative && MXUserNativeRWInit(&lock->nativeLock);
lockInited = MXRecLockInit(&lock->recursiveLock);
if (LIKELY(lockInited)) {
uint32 statsMode;
lock->holderTable = HashTable_Alloc(256,
HASH_INT_KEY | HASH_FLAG_ATOMIC,
MXUserFreeHashEntry);
statsMode = MXUserStatsMode();
switch (MXUserStatsMode()) {
case 0:
MXUserDisableStats(&lock->acquireStatsMem, &lock->heldStatsMem);
lock->header.statsFunc = NULL;
break;
case 1:
MXUserEnableStats(&lock->acquireStatsMem, NULL);
lock->header.statsFunc = MXUserStatsActionRW;
break;
case 2:
MXUserEnableStats(&lock->acquireStatsMem, &lock->heldStatsMem);
lock->header.statsFunc = MXUserStatsActionRW;
break;
default:
Panic("%s: unknown stats mode: %d!\n", __FUNCTION__, statsMode);
}
MXUserAddToList(&lock->header);
} else {
Panic("%s: native lock initialization routine failed\n", __FUNCTION__);
}
return lock;
}
