DECLINLINE(void) pdmacFileEpTaskInit(PPDMASYNCCOMPLETIONTASK pTask, size_t cbTransfer)
{
PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
Assert((uint32_t)cbTransfer == cbTransfer && (int32_t)cbTransfer >= 0);
ASMAtomicWriteS32(&pTaskFile->cbTransferLeft, (int32_t)cbTransfer);
ASMAtomicWriteBool(&pTaskFile->fCompleted, false);
ASMAtomicWriteS32(&pTaskFile->rc, VINF_SUCCESS);
}
RTDECL(int) RTCritSectDelete(PRTCRITSECT pCritSect)
{
/*
* Assert free waiters and so on.
*/
Assert(pCritSect);
Assert(pCritSect->u32Magic == RTCRITSECT_MAGIC);
Assert(pCritSect->cNestings == 0);
Assert(pCritSect->cLockers == -1);
Assert(pCritSect->NativeThreadOwner == NIL_RTNATIVETHREAD);
/*
* Invalidate the structure and free the mutex.
* In case someone is waiting we'll signal the semaphore cLockers + 1 times.
*/
ASMAtomicWriteU32(&pCritSect->u32Magic, ~RTCRITSECT_MAGIC);
pCritSect->fFlags = 0;
pCritSect->cNestings = 0;
pCritSect->NativeThreadOwner= NIL_RTNATIVETHREAD;
RTSEMEVENT EventSem = pCritSect->EventSem;
pCritSect->EventSem = NIL_RTSEMEVENT;
while (pCritSect->cLockers-- >= 0)
RTSemEventSignal(EventSem);
ASMAtomicWriteS32(&pCritSect->cLockers, -1);
int rc = RTSemEventDestroy(EventSem);
AssertRC(rc);
RTLockValidatorRecExclDestroy(&pCritSect->pValidatorRec);
return rc;
}
/**
* Clears stream error condition.
*
* All stream operations save RTStrmClose and this will fail
* while an error is asserted on the stream
*
* @returns iprt status code.
* @param pStream The stream.
*/
RTR3DECL(int) RTStrmClearError(PRTSTREAM pStream)
{
AssertReturn(RT_VALID_PTR(pStream) && pStream->u32Magic == RTSTREAM_MAGIC, VERR_INVALID_PARAMETER);
clearerr(pStream->pFile);
ASMAtomicWriteS32(&pStream->i32Error, VINF_SUCCESS);
return VINF_SUCCESS;
}
/* Note! Should *not* be exported since it's only for static linking. */
RTR0DECL(void) RTR0TermForced(void)
{
RT_ASSERT_PREEMPTIBLE();
AssertMsg(g_crtR0Users == 1, ("%d\n", g_crtR0Users));
ASMAtomicWriteS32(&g_crtR0Users, 0);
rtR0Term();
}
/**
* Reads from a file stream.
*
* @returns iprt status code.
* @param pStream The stream.
* @param pvBuf Where to put the read bits.
* Must be cbRead bytes or more.
* @param cbRead Number of bytes to read.
* @param pcbRead Where to store the number of bytes actually read.
* If NULL cbRead bytes are read or an error is returned.
*/
RTR3DECL(int) RTStrmReadEx(PRTSTREAM pStream, void *pvBuf, size_t cbRead, size_t *pcbRead)
{
AssertReturn(RT_VALID_PTR(pStream) && pStream->u32Magic == RTSTREAM_MAGIC, VERR_INVALID_PARAMETER);
int rc = pStream->i32Error;
if (RT_SUCCESS(rc))
{
if (pStream->fRecheckMode)
rtStreamRecheckMode(pStream);
if (pcbRead)
{
/*
* Can do with a partial read.
*/
*pcbRead = fread(pvBuf, 1, cbRead, pStream->pFile);
if ( *pcbRead == cbRead
|| !ferror(pStream->pFile))
return VINF_SUCCESS;
if (feof(pStream->pFile))
{
if (*pcbRead)
return VINF_EOF;
rc = VERR_EOF;
}
else if (ferror(pStream->pFile))
rc = VERR_READ_ERROR;
else
{
AssertMsgFailed(("This shouldn't happen\n"));
rc = VERR_INTERNAL_ERROR;
}
}
else
{
/*
* Must read it all!
*/
if (fread(pvBuf, cbRead, 1, pStream->pFile) == 1)
return VINF_SUCCESS;
/* possible error/eof. */
if (feof(pStream->pFile))
rc = VERR_EOF;
else if (ferror(pStream->pFile))
rc = VERR_READ_ERROR;
else
{
AssertMsgFailed(("This shouldn't happen\n"));
rc = VERR_INTERNAL_ERROR;
}
}
ASMAtomicWriteS32(&pStream->i32Error, rc);
}
return rc;
}
/**
* Rewinds the stream.
*
* Stream errors will be reset on success.
*
* @returns IPRT status code.
*
* @param pStream The stream.
*
* @remarks Not all streams are rewindable and that behavior is currently
* undefined for those.
*/
RTR3DECL(int) RTStrmRewind(PRTSTREAM pStream)
{
AssertPtrReturn(pStream, VERR_INVALID_HANDLE);
AssertReturn(pStream->u32Magic == RTSTREAM_MAGIC, VERR_INVALID_HANDLE);
int rc;
clearerr(pStream->pFile);
errno = 0;
if (!fseek(pStream->pFile, 0, SEEK_SET))
{
ASMAtomicWriteS32(&pStream->i32Error, VINF_SUCCESS);
rc = VINF_SUCCESS;
}
else
{
rc = RTErrConvertFromErrno(errno);
ASMAtomicWriteS32(&pStream->i32Error, rc);
}
return rc;
}
static int rtMpDarwinInitMaxCpus(void)
{
int32_t cCpus = -1;
size_t oldLen = sizeof(cCpus);
int rc = sysctlbyname("hw.ncpu", &cCpus, &oldLen, NULL, NULL);
if (rc)
{
printf("IPRT: sysctlbyname(hw.ncpu) failed with rc=%d!\n", rc);
cCpus = 64; /* whatever */
}
ASMAtomicWriteS32(&g_cMaxCpus, cCpus);
return cCpus;
}
/**
* Deletes one critical section.
*
* @returns Return code from RTCritSectDelete.
*
* @param pVM Pointer to the VM.
* @param pCritSect The critical section.
* @param pPrev The previous critical section in the list.
* @param fFinal Set if this is the final call and statistics shouldn't be deregistered.
*
* @remarks Caller must have entered the ListCritSect.
*/
static int pdmR3CritSectDeleteOne(PVM pVM, PUVM pUVM, PPDMCRITSECTINT pCritSect, PPDMCRITSECTINT pPrev, bool fFinal)
{
/*
* Assert free waiters and so on (c&p from RTCritSectDelete).
*/
Assert(pCritSect->Core.u32Magic == RTCRITSECT_MAGIC);
Assert(pCritSect->Core.cNestings == 0);
Assert(pCritSect->Core.cLockers == -1);
Assert(pCritSect->Core.NativeThreadOwner == NIL_RTNATIVETHREAD);
Assert(RTCritSectIsOwner(&pUVM->pdm.s.ListCritSect));
/*
* Unlink it.
*/
if (pPrev)
pPrev->pNext = pCritSect->pNext;
else
pUVM->pdm.s.pCritSects = pCritSect->pNext;
/*
* Delete it (parts taken from RTCritSectDelete).
* In case someone is waiting we'll signal the semaphore cLockers + 1 times.
*/
ASMAtomicWriteU32(&pCritSect->Core.u32Magic, 0);
SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->Core.EventSem;
pCritSect->Core.EventSem = NIL_RTSEMEVENT;
while (pCritSect->Core.cLockers-- >= 0)
SUPSemEventSignal(pVM->pSession, hEvent);
ASMAtomicWriteS32(&pCritSect->Core.cLockers, -1);
int rc = SUPSemEventClose(pVM->pSession, hEvent);
AssertRC(rc);
RTLockValidatorRecExclDestroy(&pCritSect->Core.pValidatorRec);
pCritSect->pNext = NULL;
pCritSect->pvKey = NULL;
pCritSect->pVMR3 = NULL;
pCritSect->pVMR0 = NIL_RTR0PTR;
pCritSect->pVMRC = NIL_RTRCPTR;
RTStrFree((char *)pCritSect->pszName);
pCritSect->pszName = NULL;
if (!fFinal)
{
STAMR3Deregister(pVM, &pCritSect->StatContentionRZLock);
STAMR3Deregister(pVM, &pCritSect->StatContentionRZUnlock);
STAMR3Deregister(pVM, &pCritSect->StatContentionR3);
#ifdef VBOX_WITH_STATISTICS
STAMR3Deregister(pVM, &pCritSect->StatLocked);
#endif
}
return rc;
}
RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
{
/*
* Validate.
*/
AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
return VERR_TIMER_ACTIVE;
if ( pTimer->fSpecificCpu
&& !RTMpIsCpuOnline(pTimer->idCpu))
return VERR_CPU_OFFLINE;
/*
* Start the timer.
*/
PKDPC pMasterDpc = pTimer->fOmniTimer
? &pTimer->aSubTimers[RTMpCpuIdToSetIndex(pTimer->idCpu)].NtDpc
: &pTimer->aSubTimers[0].NtDpc;
#ifndef RTR0TIMER_NT_MANUAL_RE_ARM
uint64_t u64Interval = pTimer->u64NanoInterval / 1000000; /* This is ms, believe it or not. */
ULONG ulInterval = (ULONG)u64Interval;
if (ulInterval != u64Interval)
ulInterval = MAXLONG;
else if (!ulInterval && pTimer->u64NanoInterval)
ulInterval = 1;
#endif
LARGE_INTEGER DueTime;
DueTime.QuadPart = -(int64_t)(u64First / 100); /* Relative, NT time. */
if (!DueTime.QuadPart)
DueTime.QuadPart = -1;
unsigned cSubTimers = pTimer->fOmniTimer ? pTimer->cSubTimers : 1;
for (unsigned iCpu = 0; iCpu < cSubTimers; iCpu++)
pTimer->aSubTimers[iCpu].iTick = 0;
ASMAtomicWriteS32(&pTimer->cOmniSuspendCountDown, 0);
ASMAtomicWriteBool(&pTimer->fSuspended, false);
#ifdef RTR0TIMER_NT_MANUAL_RE_ARM
pTimer->uNtStartTime = rtTimerNtQueryInterruptTime() + u64First / 100;
KeSetTimerEx(&pTimer->NtTimer, DueTime, 0, pMasterDpc);
#else
KeSetTimerEx(&pTimer->NtTimer, DueTime, ulInterval, pMasterDpc);
#endif
return VINF_SUCCESS;
}
/**
* Tail code called when we've won the battle for the lock.
*
* @returns VINF_SUCCESS.
*
* @param pCritSect The critical section.
* @param hNativeSelf The native handle of this thread.
*/
DECL_FORCE_INLINE(int) pdmCritSectEnterFirst(PPDMCRITSECT pCritSect, RTNATIVETHREAD hNativeSelf, PCRTLOCKVALSRCPOS pSrcPos)
{
AssertMsg(pCritSect->s.Core.NativeThreadOwner == NIL_RTNATIVETHREAD, ("NativeThreadOwner=%p\n", pCritSect->s.Core.NativeThreadOwner));
Assert(!(pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK));
ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1);
Assert(pCritSect->s.Core.cNestings == 1);
ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeSelf);
# ifdef PDMCRITSECT_STRICT
RTLockValidatorRecExclSetOwner(pCritSect->s.Core.pValidatorRec, NIL_RTTHREAD, pSrcPos, true);
# endif
STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l);
return VINF_SUCCESS;
}
/**
* Leaves a critical section entered with PDMCritSectEnter().
*
* @param pCritSect The PDM critical section to leave.
*/
VMMDECL(void) PDMCritSectLeave(PPDMCRITSECT pCritSect)
{
AssertMsg(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic));
Assert(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC);
/* Check for NOP sections before asserting ownership. */
if (pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP)
return;
/*
* Always check that the caller is the owner (screw performance).
*/
RTNATIVETHREAD const hNativeSelf = pdmCritSectGetNativeSelf(pCritSect);
AssertReleaseMsgReturnVoid(pCritSect->s.Core.NativeThreadOwner == hNativeSelf,
("%p %s: %p != %p; cLockers=%d cNestings=%d\n", pCritSect, R3STRING(pCritSect->s.pszName),
pCritSect->s.Core.NativeThreadOwner, hNativeSelf,
pCritSect->s.Core.cLockers, pCritSect->s.Core.cNestings));
Assert(pCritSect->s.Core.cNestings >= 1);
/*
* Nested leave.
*/
if (pCritSect->s.Core.cNestings > 1)
{
ASMAtomicDecS32(&pCritSect->s.Core.cNestings);
Assert(pCritSect->s.Core.cNestings >= 1);
ASMAtomicDecS32(&pCritSect->s.Core.cLockers);
Assert(pCritSect->s.Core.cLockers >= 0);
return;
}
#ifdef IN_RING0
# if 0 /** @todo Make SUPSemEventSignal interrupt safe (handle table++) and enable this for: defined(RT_OS_LINUX) || defined(RT_OS_OS2) */
if (1) /* SUPSemEventSignal is safe */
# else
if (ASMIntAreEnabled())
# endif
#endif
#if defined(IN_RING3) || defined(IN_RING0)
{
/*
* Leave for real.
*/
/* update members. */
# ifdef IN_RING3
RTSEMEVENT hEventToSignal = pCritSect->s.EventToSignal;
pCritSect->s.EventToSignal = NIL_RTSEMEVENT;
# if defined(PDMCRITSECT_STRICT)
if (pCritSect->s.Core.pValidatorRec->hThread != NIL_RTTHREAD)
RTLockValidatorRecExclReleaseOwnerUnchecked(pCritSect->s.Core.pValidatorRec);
# endif
Assert(!pCritSect->s.Core.pValidatorRec || pCritSect->s.Core.pValidatorRec->hThread == NIL_RTTHREAD);
# endif
ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD);
ASMAtomicDecS32(&pCritSect->s.Core.cNestings);
Assert(pCritSect->s.Core.cNestings == 0);
/* stop and decrement lockers. */
STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l);
ASMCompilerBarrier();
if (ASMAtomicDecS32(&pCritSect->s.Core.cLockers) >= 0)
{
/* Someone is waiting, wake up one of them. */
SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->s.Core.EventSem;
PSUPDRVSESSION pSession = pCritSect->s.CTX_SUFF(pVM)->pSession;
int rc = SUPSemEventSignal(pSession, hEvent);
AssertRC(rc);
}
# ifdef IN_RING3
/* Signal exit event. */
if (hEventToSignal != NIL_RTSEMEVENT)
{
LogBird(("Signalling %#x\n", hEventToSignal));
int rc = RTSemEventSignal(hEventToSignal);
AssertRC(rc);
}
# endif
# if defined(DEBUG_bird) && defined(IN_RING0)
VMMTrashVolatileXMMRegs();
# endif
}
#endif /* IN_RING3 || IN_RING0 */
#ifdef IN_RING0
else
#endif
#if defined(IN_RING0) || defined(IN_RC)
{
/*
* Try leave it.
*/
if (pCritSect->s.Core.cLockers == 0)
{
ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0);
RTNATIVETHREAD hNativeThread = pCritSect->s.Core.NativeThreadOwner;
ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l);
ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD);
if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, -1, 0))
return;
/* darn, someone raced in on us. */
ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeThread);
STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l);
Assert(pCritSect->s.Core.cNestings == 0);
ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1);
}
ASMAtomicOrU32(&pCritSect->s.Core.fFlags, PDMCRITSECT_FLAGS_PENDING_UNLOCK);
/*
* Queue the request.
*/
PVM pVM = pCritSect->s.CTX_SUFF(pVM); AssertPtr(pVM);
PVMCPU pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
uint32_t i = pVCpu->pdm.s.cQueuedCritSectLeaves++;
LogFlow(("PDMCritSectLeave: [%d]=%p => R3\n", i, pCritSect));
AssertFatal(i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectsLeaves));
pVCpu->pdm.s.apQueuedCritSectsLeaves[i] = MMHyperCCToR3(pVM, pCritSect);
VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZUnlock);
}
#endif /* IN_RING0 || IN_RC */
}
/**
* Internal write API, stream lock already held.
*
* @returns IPRT status code.
* @param pStream The stream.
* @param pvBuf What to write.
* @param cbWrite How much to write.
* @param pcbWritten Where to optionally return the number of bytes
* written.
* @param fSureIsText Set if we're sure this is UTF-8 text already.
*/
static int rtStrmWriteLocked(PRTSTREAM pStream, const void *pvBuf, size_t cbWrite, size_t *pcbWritten,
bool fSureIsText)
{
int rc = pStream->i32Error;
if (RT_FAILURE(rc))
return rc;
if (pStream->fRecheckMode)
rtStreamRecheckMode(pStream);
#ifdef RT_OS_WINDOWS
/*
* Use the unicode console API when possible in order to avoid stuff
* getting lost in unnecessary code page translations.
*/
HANDLE hCon;
if (rtStrmIsConsoleUnlocked(pStream, &hCon))
{
# ifdef HAVE_FWRITE_UNLOCKED
if (!fflush_unlocked(pStream->pFile))
# else
if (!fflush(pStream->pFile))
# endif
{
/** @todo Consider buffering later. For now, we'd rather correct output than
* fast output. */
DWORD cwcWritten = 0;
PRTUTF16 pwszSrc = NULL;
size_t cwcSrc = 0;
rc = RTStrToUtf16Ex((const char *)pvBuf, cbWrite, &pwszSrc, 0, &cwcSrc);
if (RT_SUCCESS(rc))
{
if (!WriteConsoleW(hCon, pwszSrc, (DWORD)cwcSrc, &cwcWritten, NULL))
{
/* try write char-by-char to avoid heap problem. */
cwcWritten = 0;
while (cwcWritten != cwcSrc)
{
DWORD cwcThis;
if (!WriteConsoleW(hCon, &pwszSrc[cwcWritten], 1, &cwcThis, NULL))
{
if (!pcbWritten || cwcWritten == 0)
rc = RTErrConvertFromErrno(GetLastError());
break;
}
if (cwcThis != 1) /* Unable to write current char (amount)? */
break;
cwcWritten++;
}
}
if (RT_SUCCESS(rc))
{
if (cwcWritten == cwcSrc)
{
if (pcbWritten)
*pcbWritten = cbWrite;
}
else if (pcbWritten)
{
PCRTUTF16 pwszCur = pwszSrc;
const char *pszCur = (const char *)pvBuf;
while ((uintptr_t)(pwszCur - pwszSrc) < cwcWritten)
{
RTUNICP CpIgnored;
RTUtf16GetCpEx(&pwszCur, &CpIgnored);
RTStrGetCpEx(&pszCur, &CpIgnored);
}
*pcbWritten = pszCur - (const char *)pvBuf;
}
else
rc = VERR_WRITE_ERROR;
}
RTUtf16Free(pwszSrc);
}
}
else
rc = RTErrConvertFromErrno(errno);
if (RT_FAILURE(rc))
ASMAtomicWriteS32(&pStream->i32Error, rc);
return rc;
}
#endif /* RT_OS_WINDOWS */
/*
* If we're sure it's text output, convert it from UTF-8 to the current
* code page before printing it.
*
* Note! Partial writes are not supported in this scenario because we
* cannot easily report back a written length matching the input.
*/
/** @todo Skip this if the current code set is UTF-8. */
if ( pStream->fCurrentCodeSet
&& !pStream->fBinary
&& ( fSureIsText
|| rtStrmIsUtf8Text(pvBuf, cbWrite))
)
{
char *pszSrcFree = NULL;
const char *pszSrc = (const char *)pvBuf;
if (pszSrc[cbWrite])
{
pszSrc = pszSrcFree = RTStrDupN(pszSrc, cbWrite);
if (pszSrc == NULL)
rc = VERR_NO_STR_MEMORY;
}
if (RT_SUCCESS(rc))
{
char *pszSrcCurCP;
rc = RTStrUtf8ToCurrentCP(&pszSrcCurCP, pszSrc);
if (RT_SUCCESS(rc))
{
size_t cchSrcCurCP = strlen(pszSrcCurCP);
IPRT_ALIGNMENT_CHECKS_DISABLE(); /* glibc / mempcpy again */
#ifdef HAVE_FWRITE_UNLOCKED
ssize_t cbWritten = fwrite_unlocked(pszSrcCurCP, cchSrcCurCP, 1, pStream->pFile);
#else
ssize_t cbWritten = fwrite(pszSrcCurCP, cchSrcCurCP, 1, pStream->pFile);
#endif
IPRT_ALIGNMENT_CHECKS_ENABLE();
if (cbWritten == 1)
{
if (pcbWritten)
*pcbWritten = cbWrite;
}
#ifdef HAVE_FWRITE_UNLOCKED
else if (!ferror_unlocked(pStream->pFile))
#else
else if (!ferror(pStream->pFile))
#endif
{
if (pcbWritten)
*pcbWritten = 0;
}
else
rc = VERR_WRITE_ERROR;
RTStrFree(pszSrcCurCP);
}
RTStrFree(pszSrcFree);
}
if (RT_FAILURE(rc))
ASMAtomicWriteS32(&pStream->i32Error, rc);
return rc;
}
