/** * Allocate memory from the heap. * * @returns Pointer to allocated memory. * @param pHeap Heap handle. * @param enmTag Statistics tag. Statistics are collected on a per tag * basis in addition to a global one. Thus we can easily * identify how memory is used by the VM. * @param cb Size of the block. * @param fZero Whether or not to zero the memory block. * @param pR0Ptr Where to return the ring-0 pointer. */ static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr) { if (pR0Ptr) *pR0Ptr = NIL_RTR0PTR; RTCritSectEnter(&pHeap->Lock); #ifdef MMUKHEAP_WITH_STATISTICS /* * Find/alloc statistics nodes. */ pHeap->Stat.cAllocations++; PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag); if (pStat) pStat->cAllocations++; else { pStat = (PMMUKHEAPSTAT)MMR3HeapAllocZU(pHeap->pUVM, MM_TAG_MM, sizeof(MMUKHEAPSTAT)); if (!pStat) { pHeap->Stat.cFailures++; AssertMsgFailed(("Failed to allocate heap stat record.\n")); RTCritSectLeave(&pHeap->Lock); return NULL; } pStat->Core.Key = (AVLULKEY)enmTag; RTAvlULInsert(&pHeap->pStatTree, &pStat->Core); pStat->cAllocations++; /* register the statistics */ PUVM pUVM = pHeap->pUVM; const char *pszTag = mmGetTagName(enmTag); STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/UkHeap/%s", pszTag); STAMR3RegisterFU(pUVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.", "/MM/UkHeap/%s/cAllocations", pszTag); STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.", "/MM/UkHeap/%s/cReallocations", pszTag); STAMR3RegisterFU(pUVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.", "/MM/UkHeap/%s/cFrees", pszTag); STAMR3RegisterFU(pUVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/UkHeap/%s/cFailures", pszTag); STAMR3RegisterFU(pUVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes allocated.", "/MM/UkHeap/%s/cbAllocated", pszTag); STAMR3RegisterFU(pUVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes freed.", "/MM/UkHeap/%s/cbFreed", pszTag); } #endif /* * Validate input. */ if (cb == 0) { #ifdef MMUKHEAP_WITH_STATISTICS pStat->cFailures++; pHeap->Stat.cFailures++; #endif RTCritSectLeave(&pHeap->Lock); return NULL; } /* * Allocate heap block. */ cb = RT_ALIGN_Z(cb, MMUKHEAP_SIZE_ALIGNMENT); void *pv = NULL; PMMUKHEAPSUB pSubHeapPrev = NULL; PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead; while (pSubHeap) { if (fZero) pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT); else pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT); if (pv) { /* Move the sub-heap with free memory to the head. */ if (pSubHeapPrev) { pSubHeapPrev->pNext = pSubHeap->pNext; pSubHeap->pNext = pHeap->pSubHeapHead; pHeap->pSubHeapHead = pSubHeap; } break; } pSubHeapPrev = pSubHeap; pSubHeap = pSubHeap->pNext; } if (RT_UNLIKELY(!pv)) { /* * Add another sub-heap. */ pSubHeap = mmR3UkHeapAddSubHeap(pHeap, RT_MAX(RT_ALIGN_Z(cb, PAGE_SIZE) + PAGE_SIZE * 16, _256K)); if (pSubHeap) { if (fZero) pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT); else pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT); } if (RT_UNLIKELY(!pv)) { AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cb, enmTag, &enmTag)); #ifdef MMUKHEAP_WITH_STATISTICS pStat->cFailures++; pHeap->Stat.cFailures++; #endif RTCritSectLeave(&pHeap->Lock); return NULL; } } /* * Update statistics */ #ifdef MMUKHEAP_WITH_STATISTICS size_t cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv); pStat->cbAllocated += cbActual; pStat->cbCurAllocated += cbActual; pHeap->Stat.cbAllocated += cbActual; pHeap->Stat.cbCurAllocated += cbActual; #endif if (pR0Ptr) *pR0Ptr = (uintptr_t)pv - (uintptr_t)pSubHeap->pv + pSubHeap->pvR0; RTCritSectLeave(&pHeap->Lock); return pv; }
/** * Allocate memory from the heap. * * @returns Pointer to allocated memory. * @param pHeap Heap handle. * @param enmTag Statistics tag. Statistics are collected on a per tag * basis in addition to a global one. Thus we can easily * identify how memory is used by the VM. See MM_TAG_*. * @param cbSize Size of the block. * @param fZero Whether or not to zero the memory block. */ void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero) { #ifdef MMR3HEAP_WITH_STATISTICS RTCritSectEnter(&pHeap->Lock); /* * Find/alloc statistics nodes. */ pHeap->Stat.cAllocations++; PMMHEAPSTAT pStat = (PMMHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag); if (pStat) { pStat->cAllocations++; RTCritSectLeave(&pHeap->Lock); } else { pStat = (PMMHEAPSTAT)RTMemAllocZ(sizeof(MMHEAPSTAT)); if (!pStat) { pHeap->Stat.cFailures++; AssertMsgFailed(("Failed to allocate heap stat record.\n")); RTCritSectLeave(&pHeap->Lock); return NULL; } pStat->Core.Key = (AVLULKEY)enmTag; pStat->pHeap = pHeap; RTAvlULInsert(&pHeap->pStatTree, &pStat->Core); pStat->cAllocations++; RTCritSectLeave(&pHeap->Lock); /* register the statistics */ PUVM pUVM = pHeap->pUVM; const char *pszTag = mmGetTagName(enmTag); STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Number of bytes currently allocated.", "/MM/R3Heap/%s", pszTag); STAMR3RegisterFU(pUVM, &pStat->cAllocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.", "/MM/R3Heap/%s/cAllocations", pszTag); STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.", "/MM/R3Heap/%s/cReallocations", pszTag); STAMR3RegisterFU(pUVM, &pStat->cFrees, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.", "/MM/R3Heap/%s/cFrees", pszTag); STAMR3RegisterFU(pUVM, &pStat->cFailures, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Number of failures.", "/MM/R3Heap/%s/cFailures", pszTag); STAMR3RegisterFU(pUVM, &pStat->cbAllocated, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes allocated.", "/MM/R3Heap/%s/cbAllocated", pszTag); STAMR3RegisterFU(pUVM, &pStat->cbFreed, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Total number of bytes freed.", "/MM/R3Heap/%s/cbFreed", pszTag); } #endif /* * Validate input. */ if (cbSize == 0) { #ifdef MMR3HEAP_WITH_STATISTICS RTCritSectEnter(&pHeap->Lock); pStat->cFailures++; pHeap->Stat.cFailures++; RTCritSectLeave(&pHeap->Lock); #endif return NULL; } /* * Allocate heap block. */ cbSize = RT_ALIGN_Z(cbSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR); PMMHEAPHDR pHdr = (PMMHEAPHDR)(fZero ? RTMemAllocZ(cbSize) : RTMemAlloc(cbSize)); if (!pHdr) { AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cbSize, enmTag, &enmTag)); #ifdef MMR3HEAP_WITH_STATISTICS RTCritSectEnter(&pHeap->Lock); pStat->cFailures++; pHeap->Stat.cFailures++; RTCritSectLeave(&pHeap->Lock); #endif return NULL; } Assert(!((uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1))); RTCritSectEnter(&pHeap->Lock); /* * Init and link in the header. */ pHdr->pNext = NULL; pHdr->pPrev = pHeap->pTail; if (pHdr->pPrev) pHdr->pPrev->pNext = pHdr; else pHeap->pHead = pHdr; pHeap->pTail = pHdr; #ifdef MMR3HEAP_WITH_STATISTICS pHdr->pStat = pStat; #else pHdr->pStat = &pHeap->Stat; #endif pHdr->cbSize = cbSize; /* * Update statistics */ #ifdef MMR3HEAP_WITH_STATISTICS pStat->cbAllocated += cbSize; pStat->cbCurAllocated += cbSize; pHeap->Stat.cbAllocated += cbSize; pHeap->Stat.cbCurAllocated += cbSize; #endif RTCritSectLeave(&pHeap->Lock); return pHdr + 1; }