/**
* Dumps a heap chunk to the log.
*
* @param pHeap Pointer to the heap.
* @param pCur Pointer to the chunk.
*/
static void mmHyperHeapDumpOne(PMMHYPERHEAP pHeap, PMMHYPERCHUNKFREE pCur)
{
if (MMHYPERCHUNK_ISUSED(&pCur->core))
{
if (pCur->core.offStat)
{
PMMHYPERSTAT pStat = (PMMHYPERSTAT)((uintptr_t)pCur + pCur->core.offStat);
const char *pszSelf = pCur->core.offStat == sizeof(MMHYPERCHUNK) ? " stat record" : "";
#ifdef IN_RING3
Log(("%p %06x USED offNext=%06x offPrev=-%06x %s%s\n",
pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
mmGetTagName((MMTAG)pStat->Core.Key), pszSelf));
#else
Log(("%p %06x USED offNext=%06x offPrev=-%06x %d%s\n",
pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core),
(MMTAG)pStat->Core.Key, pszSelf));
#endif
}
else
Log(("%p %06x USED offNext=%06x offPrev=-%06x\n",
pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core)));
}
else
Log(("%p %06x FREE offNext=%06x offPrev=-%06x : cb=%06x offNext=%06x offPrev=-%06x\n",
pCur, (uintptr_t)pCur - (uintptr_t)pHeap->CTX_SUFF(pbHeap),
pCur->core.offNext, -MMHYPERCHUNK_GET_OFFPREV(&pCur->core), pCur->cb, pCur->offNext, pCur->offPrev));
}
/**
* Registers statistics with STAM.
*
*/
static void mmR3HyperStatRegisterOne(PVM pVM, PMMHYPERSTAT pStat)
{
if (pStat->fRegistered)
return;
const char *pszTag = mmGetTagName((MMTAG)pStat->Core.Key);
STAMR3RegisterF(pVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/HyperHeap/%s", pszTag);
STAMR3RegisterF(pVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of alloc calls.", "/MM/HyperHeap/%s/cAllocations", pszTag);
STAMR3RegisterF(pVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of free calls.", "/MM/HyperHeap/%s/cFrees", pszTag);
STAMR3RegisterF(pVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/HyperHeap/%s/cFailures", pszTag);
STAMR3RegisterF(pVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of allocated bytes.", "/MM/HyperHeap/%s/cbAllocated", pszTag);
STAMR3RegisterF(pVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of freed bytes.", "/MM/HyperHeap/%s/cbFreed", pszTag);
STAMR3RegisterF(pVM, &pStat->cbMaxAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max number of bytes allocated at the same time.","/MM/HyperHeap/%s/cbMaxAllocated", pszTag);
pStat->fRegistered = true;
}
/**
* Wrapper for mmHyperAllocInternal
*/
VMMDECL(int) MMHyperAlloc(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
{
int rc;
rc = mmHyperLock(pVM);
AssertRCReturn(rc, rc);
LogFlow(("MMHyperAlloc %x align=%x tag=%s\n", cb, uAlignment, mmGetTagName(enmTag)));
rc = mmHyperAllocInternal(pVM, cb, uAlignment, enmTag, ppv);
mmHyperUnlock(pVM);
return rc;
}
/**
* Allocate memory from the heap.
*
* @returns Pointer to allocated memory.
* @param pHeap Heap handle.
* @param enmTag Statistics tag. Statistics are collected on a per tag
* basis in addition to a global one. Thus we can easily
* identify how memory is used by the VM.
* @param cb Size of the block.
* @param fZero Whether or not to zero the memory block.
* @param pR0Ptr Where to return the ring-0 pointer.
*/
static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr)
{
if (pR0Ptr)
*pR0Ptr = NIL_RTR0PTR;
RTCritSectEnter(&pHeap->Lock);
#ifdef MMUKHEAP_WITH_STATISTICS
/*
* Find/alloc statistics nodes.
*/
pHeap->Stat.cAllocations++;
PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
if (pStat)
pStat->cAllocations++;
else
{
pStat = (PMMUKHEAPSTAT)MMR3HeapAllocZU(pHeap->pUVM, MM_TAG_MM, sizeof(MMUKHEAPSTAT));
if (!pStat)
{
pHeap->Stat.cFailures++;
AssertMsgFailed(("Failed to allocate heap stat record.\n"));
RTCritSectLeave(&pHeap->Lock);
return NULL;
}
pStat->Core.Key = (AVLULKEY)enmTag;
RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
pStat->cAllocations++;
/* register the statistics */
PUVM pUVM = pHeap->pUVM;
const char *pszTag = mmGetTagName(enmTag);
STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/UkHeap/%s", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.", "/MM/UkHeap/%s/cAllocations", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.", "/MM/UkHeap/%s/cReallocations", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.", "/MM/UkHeap/%s/cFrees", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/UkHeap/%s/cFailures", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes allocated.", "/MM/UkHeap/%s/cbAllocated", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes freed.", "/MM/UkHeap/%s/cbFreed", pszTag);
}
#endif
/*
* Validate input.
*/
if (cb == 0)
{
#ifdef MMUKHEAP_WITH_STATISTICS
pStat->cFailures++;
pHeap->Stat.cFailures++;
#endif
RTCritSectLeave(&pHeap->Lock);
return NULL;
}
/*
* Allocate heap block.
*/
cb = RT_ALIGN_Z(cb, MMUKHEAP_SIZE_ALIGNMENT);
void *pv = NULL;
PMMUKHEAPSUB pSubHeapPrev = NULL;
PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
while (pSubHeap)
{
if (fZero)
pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
else
pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
if (pv)
{
/* Move the sub-heap with free memory to the head. */
if (pSubHeapPrev)
{
pSubHeapPrev->pNext = pSubHeap->pNext;
pSubHeap->pNext = pHeap->pSubHeapHead;
pHeap->pSubHeapHead = pSubHeap;
}
break;
}
pSubHeapPrev = pSubHeap;
pSubHeap = pSubHeap->pNext;
}
if (RT_UNLIKELY(!pv))
{
/*
* Add another sub-heap.
*/
pSubHeap = mmR3UkHeapAddSubHeap(pHeap, RT_MAX(RT_ALIGN_Z(cb, PAGE_SIZE) + PAGE_SIZE * 16, _256K));
if (pSubHeap)
{
if (fZero)
pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
else
pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
}
if (RT_UNLIKELY(!pv))
{
AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cb, enmTag, &enmTag));
#ifdef MMUKHEAP_WITH_STATISTICS
pStat->cFailures++;
pHeap->Stat.cFailures++;
#endif
RTCritSectLeave(&pHeap->Lock);
return NULL;
}
}
/*
* Update statistics
*/
#ifdef MMUKHEAP_WITH_STATISTICS
size_t cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
pStat->cbAllocated += cbActual;
pStat->cbCurAllocated += cbActual;
pHeap->Stat.cbAllocated += cbActual;
pHeap->Stat.cbCurAllocated += cbActual;
#endif
if (pR0Ptr)
*pR0Ptr = (uintptr_t)pv - (uintptr_t)pSubHeap->pv + pSubHeap->pvR0;
RTCritSectLeave(&pHeap->Lock);
return pv;
}
/**
* Allocate memory from the heap.
*
* @returns Pointer to allocated memory.
* @param pHeap Heap handle.
* @param enmTag Statistics tag. Statistics are collected on a per tag
* basis in addition to a global one. Thus we can easily
* identify how memory is used by the VM. See MM_TAG_*.
* @param cbSize Size of the block.
* @param fZero Whether or not to zero the memory block.
*/
void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero)
{
#ifdef MMR3HEAP_WITH_STATISTICS
RTCritSectEnter(&pHeap->Lock);
/*
* Find/alloc statistics nodes.
*/
pHeap->Stat.cAllocations++;
PMMHEAPSTAT pStat = (PMMHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
if (pStat)
{
pStat->cAllocations++;
RTCritSectLeave(&pHeap->Lock);
}
else
{
pStat = (PMMHEAPSTAT)RTMemAllocZ(sizeof(MMHEAPSTAT));
if (!pStat)
{
pHeap->Stat.cFailures++;
AssertMsgFailed(("Failed to allocate heap stat record.\n"));
RTCritSectLeave(&pHeap->Lock);
return NULL;
}
pStat->Core.Key = (AVLULKEY)enmTag;
pStat->pHeap = pHeap;
RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
pStat->cAllocations++;
RTCritSectLeave(&pHeap->Lock);
/* register the statistics */
PUVM pUVM = pHeap->pUVM;
const char *pszTag = mmGetTagName(enmTag);
STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/R3Heap/%s", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.", "/MM/R3Heap/%s/cAllocations", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.", "/MM/R3Heap/%s/cReallocations", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.", "/MM/R3Heap/%s/cFrees", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/R3Heap/%s/cFailures", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes allocated.", "/MM/R3Heap/%s/cbAllocated", pszTag);
STAMR3RegisterFU(pUVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes freed.", "/MM/R3Heap/%s/cbFreed", pszTag);
}
#endif
/*
* Validate input.
*/
if (cbSize == 0)
{
#ifdef MMR3HEAP_WITH_STATISTICS
RTCritSectEnter(&pHeap->Lock);
pStat->cFailures++;
pHeap->Stat.cFailures++;
RTCritSectLeave(&pHeap->Lock);
#endif
return NULL;
}
/*
* Allocate heap block.
*/
cbSize = RT_ALIGN_Z(cbSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
PMMHEAPHDR pHdr = (PMMHEAPHDR)(fZero ? RTMemAllocZ(cbSize) : RTMemAlloc(cbSize));
if (!pHdr)
{
AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cbSize, enmTag, &enmTag));
#ifdef MMR3HEAP_WITH_STATISTICS
RTCritSectEnter(&pHeap->Lock);
pStat->cFailures++;
pHeap->Stat.cFailures++;
RTCritSectLeave(&pHeap->Lock);
#endif
return NULL;
}
Assert(!((uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1)));
RTCritSectEnter(&pHeap->Lock);
/*
* Init and link in the header.
*/
pHdr->pNext = NULL;
pHdr->pPrev = pHeap->pTail;
if (pHdr->pPrev)
pHdr->pPrev->pNext = pHdr;
else
pHeap->pHead = pHdr;
pHeap->pTail = pHdr;
#ifdef MMR3HEAP_WITH_STATISTICS
pHdr->pStat = pStat;
#else
pHdr->pStat = &pHeap->Stat;
#endif
pHdr->cbSize = cbSize;
/*
* Update statistics
*/
#ifdef MMR3HEAP_WITH_STATISTICS
pStat->cbAllocated += cbSize;
pStat->cbCurAllocated += cbSize;
pHeap->Stat.cbAllocated += cbSize;
pHeap->Stat.cbCurAllocated += cbSize;
#endif
RTCritSectLeave(&pHeap->Lock);
return pHdr + 1;
}
