/**
* Allocate memory from the heap.
*
* @returns Pointer to allocated memory.
* @param pHeap Heap handle.
* @param enmTag Statistics tag. Statistics are collected on a per tag
* basis in addition to a global one. Thus we can easily
* identify how memory is used by the VM.
* @param cb Size of the block.
* @param fZero Whether or not to zero the memory block.
* @param pR0Ptr Where to return the ring-0 pointer.
*/
static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr)
{
if (pR0Ptr)
*pR0Ptr = NIL_RTR0PTR;
RTCritSectEnter(&pHeap->Lock);
#ifdef MMUKHEAP_WITH_STATISTICS
/*
* Find/alloc statistics nodes.
*/
pHeap->Stat.cAllocations++;
PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
if (pStat)
pStat->cAllocations++;
else
{
pStat = (PMMUKHEAPSTAT)MMR3HeapAllocZU(pHeap->pUVM, MM_TAG_MM, sizeof(MMUKHEAPSTAT));
if (!pStat)
{
pHeap->Stat.cFailures++;
AssertMsgFailed(("Failed to allocate heap stat record.\n"));
RTCritSectLeave(&pHeap->Lock);
return NULL;
}
pStat->Core.Key = (AVLULKEY)enmTag;
RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
pStat->cAllocations++;
/* register the statistics */
PUVM pUVM = pHeap->pUVM;
const char *pszTag = mmGetTagName(enmTag);
STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/UkHeap/%s", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.", "/MM/UkHeap/%s/cAllocations", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.", "/MM/UkHeap/%s/cReallocations", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.", "/MM/UkHeap/%s/cFrees", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/UkHeap/%s/cFailures", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes allocated.", "/MM/UkHeap/%s/cbAllocated", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes freed.", "/MM/UkHeap/%s/cbFreed", pszTag);
}
#endif
/*
* Validate input.
*/
if (cb == 0)
{
#ifdef MMUKHEAP_WITH_STATISTICS
pStat->cFailures++;
pHeap->Stat.cFailures++;
#endif
RTCritSectLeave(&pHeap->Lock);
return NULL;
}
/*
* Allocate heap block.
*/
cb = RT_ALIGN_Z(cb, MMUKHEAP_SIZE_ALIGNMENT);
void *pv = NULL;
PMMUKHEAPSUB pSubHeapPrev = NULL;
PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
while (pSubHeap)
{
if (fZero)
pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
else
pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
if (pv)
{
/* Move the sub-heap with free memory to the head. */
if (pSubHeapPrev)
{
pSubHeapPrev->pNext = pSubHeap->pNext;
pSubHeap->pNext = pHeap->pSubHeapHead;
pHeap->pSubHeapHead = pSubHeap;
}
break;
}
pSubHeapPrev = pSubHeap;
pSubHeap = pSubHeap->pNext;
}
if (RT_UNLIKELY(!pv))
{
/*
* Add another sub-heap.
*/
pSubHeap = mmR3UkHeapAddSubHeap(pHeap, RT_MAX(RT_ALIGN_Z(cb, PAGE_SIZE) + PAGE_SIZE * 16, _256K));
if (pSubHeap)
{
if (fZero)
pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
else
pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
}
if (RT_UNLIKELY(!pv))
{
AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cb, enmTag, &enmTag));
#ifdef MMUKHEAP_WITH_STATISTICS
pStat->cFailures++;
pHeap->Stat.cFailures++;
#endif
RTCritSectLeave(&pHeap->Lock);
return NULL;
}
}
/*
* Update statistics
*/
#ifdef MMUKHEAP_WITH_STATISTICS
size_t cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
pStat->cbAllocated += cbActual;
pStat->cbCurAllocated += cbActual;
pHeap->Stat.cbAllocated += cbActual;
pHeap->Stat.cbCurAllocated += cbActual;
#endif
if (pR0Ptr)
*pR0Ptr = (uintptr_t)pv - (uintptr_t)pSubHeap->pv + pSubHeap->pvR0;
RTCritSectLeave(&pHeap->Lock);
return pv;
}
uint32_t hgcmObjMake (HGCMObject *pObject, uint32_t u32HandleIn)
{
int handle = 0;
LogFlow(("MAIN::hgcmObjGenerateHandle: pObject %p\n", pObject));
int rc = hgcmObjEnter ();
if (RT_SUCCESS(rc))
{
ObjectAVLCore *pCore = &pObject->m_core;
/* Generate a new handle value. */
uint32_t volatile *pu32HandleCountSource = pObject->Type () == HGCMOBJ_CLIENT?
&g_u32ClientHandleCount:
&g_u32InternalHandleCount;
uint32_t u32Start = *pu32HandleCountSource;
for (;;)
{
uint32_t Key;
if (u32HandleIn == 0)
{
Key = ASMAtomicIncU32 (pu32HandleCountSource);
if (Key == u32Start)
{
/* Rollover. Something is wrong. */
AssertReleaseFailed ();
break;
}
/* 0 and 0x80000000 are not valid handles. */
if ((Key & 0x7FFFFFFF) == 0)
{
/* Over the invalid value, reinitialize the source. */
*pu32HandleCountSource = pObject->Type () == HGCMOBJ_CLIENT?
0:
0x80000000;
continue;
}
}
else
{
Key = u32HandleIn;
}
/* Insert object to AVL tree. */
pCore->AvlCore.Key = Key;
bool bRC = RTAvlULInsert(&g_pTree, &pCore->AvlCore);
/* Could not insert a handle. */
if (!bRC)
{
if (u32HandleIn == 0)
{
/* Try another generated handle. */
continue;
}
else
{
/* Could not use the specified handle. */
break;
}
}
/* Initialize backlink. */
pCore->pSelf = pObject;
/* Reference the object for time while it resides in the tree. */
pObject->Reference ();
/* Store returned handle. */
handle = Key;
Log(("Object key inserted 0x%08X\n", Key));
break;
}
hgcmObjLeave ();
}
else
{
AssertReleaseMsgFailed (("MAIN::hgcmObjGenerateHandle: Failed to acquire object pool semaphore"));
}
LogFlow(("MAIN::hgcmObjGenerateHandle: handle = 0x%08X, rc = %Rrc, return void\n", handle, rc));
return handle;
}
/**
* Allocate memory from the heap.
*
* @returns Pointer to allocated memory.
* @param pHeap Heap handle.
* @param enmTag Statistics tag. Statistics are collected on a per tag
* basis in addition to a global one. Thus we can easily
* identify how memory is used by the VM. See MM_TAG_*.
* @param cbSize Size of the block.
* @param fZero Whether or not to zero the memory block.
*/
void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero)
{
#ifdef MMR3HEAP_WITH_STATISTICS
RTCritSectEnter(&pHeap->Lock);
/*
* Find/alloc statistics nodes.
*/
pHeap->Stat.cAllocations++;
PMMHEAPSTAT pStat = (PMMHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
if (pStat)
{
pStat->cAllocations++;
RTCritSectLeave(&pHeap->Lock);
}
else
{
pStat = (PMMHEAPSTAT)RTMemAllocZ(sizeof(MMHEAPSTAT));
if (!pStat)
{
pHeap->Stat.cFailures++;
AssertMsgFailed(("Failed to allocate heap stat record.\n"));
RTCritSectLeave(&pHeap->Lock);
return NULL;
}
pStat->Core.Key = (AVLULKEY)enmTag;
pStat->pHeap = pHeap;
RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
pStat->cAllocations++;
RTCritSectLeave(&pHeap->Lock);
/* register the statistics */
PUVM pUVM = pHeap->pUVM;
const char *pszTag = mmGetTagName(enmTag);
STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/R3Heap/%s", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.", "/MM/R3Heap/%s/cAllocations", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.", "/MM/R3Heap/%s/cReallocations", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.", "/MM/R3Heap/%s/cFrees", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/R3Heap/%s/cFailures", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes allocated.", "/MM/R3Heap/%s/cbAllocated", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes freed.", "/MM/R3Heap/%s/cbFreed", pszTag);
}
#endif
/*
* Validate input.
*/
if (cbSize == 0)
{
#ifdef MMR3HEAP_WITH_STATISTICS
RTCritSectEnter(&pHeap->Lock);
pStat->cFailures++;
pHeap->Stat.cFailures++;
RTCritSectLeave(&pHeap->Lock);
#endif
return NULL;
}
/*
* Allocate heap block.
*/
cbSize = RT_ALIGN_Z(cbSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
PMMHEAPHDR pHdr = (PMMHEAPHDR)(fZero ? RTMemAllocZ(cbSize) : RTMemAlloc(cbSize));
if (!pHdr)
{
AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cbSize, enmTag, &enmTag));
#ifdef MMR3HEAP_WITH_STATISTICS
RTCritSectEnter(&pHeap->Lock);
pStat->cFailures++;
pHeap->Stat.cFailures++;
RTCritSectLeave(&pHeap->Lock);
#endif
return NULL;
}
Assert(!((uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1)));
RTCritSectEnter(&pHeap->Lock);
/*
* Init and link in the header.
*/
pHdr->pNext = NULL;
pHdr->pPrev = pHeap->pTail;
if (pHdr->pPrev)
pHdr->pPrev->pNext = pHdr;
else
pHeap->pHead = pHdr;
pHeap->pTail = pHdr;
#ifdef MMR3HEAP_WITH_STATISTICS
pHdr->pStat = pStat;
#else
pHdr->pStat = &pHeap->Stat;
#endif
pHdr->cbSize = cbSize;
/*
* Update statistics
*/
#ifdef MMR3HEAP_WITH_STATISTICS
pStat->cbAllocated += cbSize;
pStat->cbCurAllocated += cbSize;
pHeap->Stat.cbAllocated += cbSize;
pHeap->Stat.cbCurAllocated += cbSize;
#endif
RTCritSectLeave(&pHeap->Lock);
return pHdr + 1;
}
