/** * Internal consistency check (relying on assertions). * @param pHeapInt */ static void rtHeapOffsetAssertAll(PRTHEAPOFFSETINTERNAL pHeapInt) { PRTHEAPOFFSETFREE pPrev = NULL; PRTHEAPOFFSETFREE pPrevFree = NULL; PRTHEAPOFFSETFREE pBlock; for (pBlock = (PRTHEAPOFFSETFREE)(pHeapInt + 1); pBlock; pBlock = RTHEAPOFF_TO_PTR_N(pHeapInt, pBlock->Core.offNext, PRTHEAPOFFSETFREE)) { if (RTHEAPOFFSETBLOCK_IS_FREE(&pBlock->Core)) { ASSERT_BLOCK_FREE(pHeapInt, pBlock); Assert(pBlock->offPrev == RTHEAPOFF_TO_OFF(pHeapInt, pPrevFree)); Assert(pPrevFree || pHeapInt->offFreeHead == RTHEAPOFF_TO_OFF(pHeapInt, pBlock)); pPrevFree = pBlock; } else ASSERT_BLOCK_USED(pHeapInt, &pBlock->Core); Assert(!pPrev || RTHEAPOFF_TO_OFF(pHeapInt, pPrev) == pBlock->Core.offPrev); pPrev = pBlock; } Assert(pHeapInt->offFreeTail == RTHEAPOFF_TO_OFF(pHeapInt, pPrevFree)); }
/** * Internal consistency check (relying on assertions). * @param pHeapInt */ static void rtHeapSimpleAssertAll(PRTHEAPSIMPLEINTERNAL pHeapInt) { PRTHEAPSIMPLEFREE pPrev = NULL; PRTHEAPSIMPLEFREE pPrevFree = NULL; PRTHEAPSIMPLEFREE pBlock; for (pBlock = (PRTHEAPSIMPLEFREE)(pHeapInt + 1); pBlock; pBlock = (PRTHEAPSIMPLEFREE)pBlock->Core.pNext) { if (RTHEAPSIMPLEBLOCK_IS_FREE(&pBlock->Core)) { ASSERT_BLOCK_FREE(pHeapInt, pBlock); Assert(pBlock->pPrev == pPrevFree); Assert(pPrevFree || pHeapInt->pFreeHead == pBlock); pPrevFree = pBlock; } else ASSERT_BLOCK_USED(pHeapInt, &pBlock->Core); Assert(!pPrev || pPrev == (PRTHEAPSIMPLEFREE)pBlock->Core.pPrev); pPrev = pBlock; } Assert(pHeapInt->pFreeTail == pPrevFree); }
/** * Free a memory block. * * @param pHeapInt The heap. * @param pBlock The memory block to free. */ static void rtHeapOffsetFreeBlock(PRTHEAPOFFSETINTERNAL pHeapInt, PRTHEAPOFFSETBLOCK pBlock) { PRTHEAPOFFSETFREE pFree = (PRTHEAPOFFSETFREE)pBlock; PRTHEAPOFFSETFREE pLeft; PRTHEAPOFFSETFREE pRight; #ifdef RTHEAPOFFSET_STRICT rtHeapOffsetAssertAll(pHeapInt); #endif /* * Look for the closest free list blocks by walking the blocks right * of us (both lists are sorted by address). */ pLeft = NULL; pRight = NULL; if (pHeapInt->offFreeTail) { pRight = RTHEAPOFF_TO_PTR_N(pHeapInt, pFree->Core.offNext, PRTHEAPOFFSETFREE); while (pRight && !RTHEAPOFFSETBLOCK_IS_FREE(&pRight->Core)) { ASSERT_BLOCK(pHeapInt, &pRight->Core); pRight = RTHEAPOFF_TO_PTR_N(pHeapInt, pRight->Core.offNext, PRTHEAPOFFSETFREE); } if (!pRight) pLeft = RTHEAPOFF_TO_PTR_N(pHeapInt, pHeapInt->offFreeTail, PRTHEAPOFFSETFREE); else { ASSERT_BLOCK_FREE(pHeapInt, pRight); pLeft = RTHEAPOFF_TO_PTR_N(pHeapInt, pRight->offPrev, PRTHEAPOFFSETFREE); } if (pLeft) ASSERT_BLOCK_FREE(pHeapInt, pLeft); } AssertMsgReturnVoid(pLeft != pFree, ("Freed twice! pv=%p (pBlock=%p)\n", pBlock + 1, pBlock)); ASSERT_L(RTHEAPOFF_TO_OFF(pHeapInt, pLeft), RTHEAPOFF_TO_OFF(pHeapInt, pFree)); Assert(!pRight || (uintptr_t)pRight > (uintptr_t)pFree); Assert(!pLeft || RTHEAPOFF_TO_PTR_N(pHeapInt, pLeft->offNext, PRTHEAPOFFSETFREE) == pRight); /* * Insert at the head of the free block list? */ if (!pLeft) { Assert(pRight == RTHEAPOFF_TO_PTR_N(pHeapInt, pHeapInt->offFreeHead, PRTHEAPOFFSETFREE)); pFree->Core.fFlags |= RTHEAPOFFSETBLOCK_FLAGS_FREE; pFree->offPrev = 0; pFree->offNext = RTHEAPOFF_TO_OFF(pHeapInt, pRight); if (pRight) pRight->offPrev = RTHEAPOFF_TO_OFF(pHeapInt, pFree); else pHeapInt->offFreeTail = RTHEAPOFF_TO_OFF(pHeapInt, pFree); pHeapInt->offFreeHead = RTHEAPOFF_TO_OFF(pHeapInt, pFree); } else { /* * Can we merge with left hand free block? */ if (pLeft->Core.offNext == RTHEAPOFF_TO_OFF(pHeapInt, pFree)) { pLeft->Core.offNext = pFree->Core.offNext; if (pFree->Core.offNext) RTHEAPOFF_TO_PTR(pHeapInt, pFree->Core.offNext, PRTHEAPOFFSETBLOCK)->offPrev = RTHEAPOFF_TO_OFF(pHeapInt, pLeft); pHeapInt->cbFree -= pLeft->cb; pFree = pLeft; } /* * No, just link it into the free list then. */ else { pFree->Core.fFlags |= RTHEAPOFFSETBLOCK_FLAGS_FREE; pFree->offNext = RTHEAPOFF_TO_OFF(pHeapInt, pRight); pFree->offPrev = RTHEAPOFF_TO_OFF(pHeapInt, pLeft); pLeft->offNext = RTHEAPOFF_TO_OFF(pHeapInt, pFree); if (pRight) pRight->offPrev = RTHEAPOFF_TO_OFF(pHeapInt, pFree); else pHeapInt->offFreeTail = RTHEAPOFF_TO_OFF(pHeapInt, pFree); } } /* * Can we merge with right hand free block? */ if ( pRight && pRight->Core.offPrev == RTHEAPOFF_TO_OFF(pHeapInt, pFree)) { /* core */ pFree->Core.offNext = pRight->Core.offNext; if (pRight->Core.offNext) RTHEAPOFF_TO_PTR(pHeapInt, pRight->Core.offNext, PRTHEAPOFFSETBLOCK)->offPrev = RTHEAPOFF_TO_OFF(pHeapInt, pFree); /* free */ pFree->offNext = pRight->offNext; if (pRight->offNext) RTHEAPOFF_TO_PTR(pHeapInt, pRight->offNext, PRTHEAPOFFSETFREE)->offPrev = RTHEAPOFF_TO_OFF(pHeapInt, pFree); else pHeapInt->offFreeTail = RTHEAPOFF_TO_OFF(pHeapInt, pFree); pHeapInt->cbFree -= pRight->cb; } /* * Calculate the size and update free stats. */ pFree->cb = (pFree->Core.offNext ? pFree->Core.offNext : pHeapInt->cbHeap) - RTHEAPOFF_TO_OFF(pHeapInt, pFree) - sizeof(RTHEAPOFFSETBLOCK); pHeapInt->cbFree += pFree->cb; ASSERT_BLOCK_FREE(pHeapInt, pFree); #ifdef RTHEAPOFFSET_STRICT rtHeapOffsetAssertAll(pHeapInt); #endif }
/** * Allocates a block of memory from the specified heap. * * No parameter validation or adjustment is performed. * * @returns Pointer to the allocated block. * @returns NULL on failure. * * @param pHeapInt The heap. * @param cb Size of the memory block to allocate. * @param uAlignment The alignment specifications for the allocated block. */ static PRTHEAPOFFSETBLOCK rtHeapOffsetAllocBlock(PRTHEAPOFFSETINTERNAL pHeapInt, size_t cb, size_t uAlignment) { PRTHEAPOFFSETBLOCK pRet = NULL; PRTHEAPOFFSETFREE pFree; AssertReturn((pHeapInt)->u32Magic == RTHEAPOFFSET_MAGIC, NULL); #ifdef RTHEAPOFFSET_STRICT rtHeapOffsetAssertAll(pHeapInt); #endif /* * Search for a fitting block from the lower end of the heap. */ for (pFree = RTHEAPOFF_TO_PTR_N(pHeapInt, pHeapInt->offFreeHead, PRTHEAPOFFSETFREE); pFree; pFree = RTHEAPOFF_TO_PTR_N(pHeapInt, pFree->offNext, PRTHEAPOFFSETFREE)) { uintptr_t offAlign; ASSERT_BLOCK_FREE(pHeapInt, pFree); /* * Match for size and alignment. */ if (pFree->cb < cb) continue; offAlign = (uintptr_t)(&pFree->Core + 1) & (uAlignment - 1); if (offAlign) { PRTHEAPOFFSETFREE pPrev; offAlign = (uintptr_t)(&pFree[1].Core + 1) & (uAlignment - 1); offAlign = uAlignment - offAlign; if (pFree->cb < cb + offAlign + sizeof(RTHEAPOFFSETFREE)) continue; /* * Split up the free block into two, so that the 2nd is aligned as * per specification. */ pPrev = pFree; pFree = (PRTHEAPOFFSETFREE)((uintptr_t)(pFree + 1) + offAlign); pFree->Core.offPrev = pPrev->Core.offSelf; pFree->Core.offNext = pPrev->Core.offNext; pFree->Core.offSelf = RTHEAPOFF_TO_OFF(pHeapInt, pFree); pFree->Core.fFlags = RTHEAPOFFSETBLOCK_FLAGS_MAGIC | RTHEAPOFFSETBLOCK_FLAGS_FREE; pFree->offPrev = pPrev->Core.offSelf; pFree->offNext = pPrev->offNext; pFree->cb = (pFree->Core.offNext ? pFree->Core.offNext : pHeapInt->cbHeap) - pFree->Core.offSelf - sizeof(RTHEAPOFFSETBLOCK); pPrev->Core.offNext = pFree->Core.offSelf; pPrev->offNext = pFree->Core.offSelf; pPrev->cb = pFree->Core.offSelf - pPrev->Core.offSelf - sizeof(RTHEAPOFFSETBLOCK); if (pFree->Core.offNext) RTHEAPOFF_TO_PTR(pHeapInt, pFree->Core.offNext, PRTHEAPOFFSETBLOCK)->offPrev = pFree->Core.offSelf; if (pFree->offNext) RTHEAPOFF_TO_PTR(pHeapInt, pFree->Core.offNext, PRTHEAPOFFSETFREE)->offPrev = pFree->Core.offSelf; else pHeapInt->offFreeTail = pFree->Core.offSelf; pHeapInt->cbFree -= sizeof(RTHEAPOFFSETBLOCK); ASSERT_BLOCK_FREE(pHeapInt, pPrev); ASSERT_BLOCK_FREE(pHeapInt, pFree); } /* * Split off a new FREE block? */ if (pFree->cb >= cb + RT_ALIGN_Z(sizeof(RTHEAPOFFSETFREE), RTHEAPOFFSET_ALIGNMENT)) { /* * Create a new FREE block at then end of this one. */ PRTHEAPOFFSETFREE pNew = (PRTHEAPOFFSETFREE)((uintptr_t)&pFree->Core + cb + sizeof(RTHEAPOFFSETBLOCK)); pNew->Core.offSelf = RTHEAPOFF_TO_OFF(pHeapInt, pNew); pNew->Core.offNext = pFree->Core.offNext; if (pFree->Core.offNext) RTHEAPOFF_TO_PTR(pHeapInt, pFree->Core.offNext, PRTHEAPOFFSETBLOCK)->offPrev = pNew->Core.offSelf; pNew->Core.offPrev = RTHEAPOFF_TO_OFF(pHeapInt, pFree); pNew->Core.fFlags = RTHEAPOFFSETBLOCK_FLAGS_MAGIC | RTHEAPOFFSETBLOCK_FLAGS_FREE; pNew->offNext = pFree->offNext; if (pNew->offNext) RTHEAPOFF_TO_PTR(pHeapInt, pNew->offNext, PRTHEAPOFFSETFREE)->offPrev = pNew->Core.offSelf; else pHeapInt->offFreeTail = pNew->Core.offSelf; pNew->offPrev = pFree->offPrev; if (pNew->offPrev) RTHEAPOFF_TO_PTR(pHeapInt, pNew->offPrev, PRTHEAPOFFSETFREE)->offNext = pNew->Core.offSelf; else pHeapInt->offFreeHead = pNew->Core.offSelf; pNew->cb = (pNew->Core.offNext ? pNew->Core.offNext : pHeapInt->cbHeap) \ - pNew->Core.offSelf - sizeof(RTHEAPOFFSETBLOCK); ASSERT_BLOCK_FREE(pHeapInt, pNew); /* * Adjust and convert the old FREE node into a USED node. */ pFree->Core.fFlags &= ~RTHEAPOFFSETBLOCK_FLAGS_FREE; pFree->Core.offNext = pNew->Core.offSelf; pHeapInt->cbFree -= pFree->cb; pHeapInt->cbFree += pNew->cb; pRet = &pFree->Core; ASSERT_BLOCK_USED(pHeapInt, pRet); } else { /* * Link it out of the free list. */ if (pFree->offNext) RTHEAPOFF_TO_PTR(pHeapInt, pFree->offNext, PRTHEAPOFFSETFREE)->offPrev = pFree->offPrev; else pHeapInt->offFreeTail = pFree->offPrev; if (pFree->offPrev) RTHEAPOFF_TO_PTR(pHeapInt, pFree->offPrev, PRTHEAPOFFSETFREE)->offNext = pFree->offNext; else pHeapInt->offFreeHead = pFree->offNext; /* * Convert it to a used block. */ pHeapInt->cbFree -= pFree->cb; pFree->Core.fFlags &= ~RTHEAPOFFSETBLOCK_FLAGS_FREE; pRet = &pFree->Core; ASSERT_BLOCK_USED(pHeapInt, pRet); } break; } #ifdef RTHEAPOFFSET_STRICT rtHeapOffsetAssertAll(pHeapInt); #endif return pRet; }
/** * Free a memory block. * * @param pHeapInt The heap. * @param pBlock The memory block to free. */ static void rtHeapSimpleFreeBlock(PRTHEAPSIMPLEINTERNAL pHeapInt, PRTHEAPSIMPLEBLOCK pBlock) { PRTHEAPSIMPLEFREE pFree = (PRTHEAPSIMPLEFREE)pBlock; PRTHEAPSIMPLEFREE pLeft; PRTHEAPSIMPLEFREE pRight; #ifdef RTHEAPSIMPLE_STRICT rtHeapSimpleAssertAll(pHeapInt); #endif /* * Look for the closest free list blocks by walking the blocks right * of us (both lists are sorted by address). */ pLeft = NULL; pRight = NULL; if (pHeapInt->pFreeTail) { pRight = (PRTHEAPSIMPLEFREE)pFree->Core.pNext; while (pRight && !RTHEAPSIMPLEBLOCK_IS_FREE(&pRight->Core)) { ASSERT_BLOCK(pHeapInt, &pRight->Core); pRight = (PRTHEAPSIMPLEFREE)pRight->Core.pNext; } if (!pRight) pLeft = pHeapInt->pFreeTail; else { ASSERT_BLOCK_FREE(pHeapInt, pRight); pLeft = pRight->pPrev; } if (pLeft) ASSERT_BLOCK_FREE(pHeapInt, pLeft); } AssertMsgReturnVoid(pLeft != pFree, ("Freed twice! pv=%p (pBlock=%p)\n", pBlock + 1, pBlock)); ASSERT_L(pLeft, pFree); Assert(!pRight || (uintptr_t)pRight > (uintptr_t)pFree); Assert(!pLeft || pLeft->pNext == pRight); /* * Insert at the head of the free block list? */ if (!pLeft) { Assert(pRight == pHeapInt->pFreeHead); pFree->Core.fFlags |= RTHEAPSIMPLEBLOCK_FLAGS_FREE; pFree->pPrev = NULL; pFree->pNext = pRight; if (pRight) pRight->pPrev = pFree; else pHeapInt->pFreeTail = pFree; pHeapInt->pFreeHead = pFree; } else { /* * Can we merge with left hand free block? */ if (pLeft->Core.pNext == &pFree->Core) { pLeft->Core.pNext = pFree->Core.pNext; if (pFree->Core.pNext) pFree->Core.pNext->pPrev = &pLeft->Core; pHeapInt->cbFree -= pLeft->cb; pFree = pLeft; } /* * No, just link it into the free list then. */ else { pFree->Core.fFlags |= RTHEAPSIMPLEBLOCK_FLAGS_FREE; pFree->pNext = pRight; pFree->pPrev = pLeft; pLeft->pNext = pFree; if (pRight) pRight->pPrev = pFree; else pHeapInt->pFreeTail = pFree; } } /* * Can we merge with right hand free block? */ if ( pRight && pRight->Core.pPrev == &pFree->Core) { /* core */ pFree->Core.pNext = pRight->Core.pNext; if (pRight->Core.pNext) pRight->Core.pNext->pPrev = &pFree->Core; /* free */ pFree->pNext = pRight->pNext; if (pRight->pNext) pRight->pNext->pPrev = pFree; else pHeapInt->pFreeTail = pFree; pHeapInt->cbFree -= pRight->cb; } /* * Calculate the size and update free stats. */ pFree->cb = (pFree->Core.pNext ? (uintptr_t)pFree->Core.pNext : (uintptr_t)pHeapInt->pvEnd) - (uintptr_t)pFree - sizeof(RTHEAPSIMPLEBLOCK); pHeapInt->cbFree += pFree->cb; ASSERT_BLOCK_FREE(pHeapInt, pFree); #ifdef RTHEAPSIMPLE_STRICT rtHeapSimpleAssertAll(pHeapInt); #endif }
/** * Allocates a block of memory from the specified heap. * * No parameter validation or adjustment is performed. * * @returns Pointer to the allocated block. * @returns NULL on failure. * * @param pHeapInt The heap. * @param cb Size of the memory block to allocate. * @param uAlignment The alignment specifications for the allocated block. */ static PRTHEAPSIMPLEBLOCK rtHeapSimpleAllocBlock(PRTHEAPSIMPLEINTERNAL pHeapInt, size_t cb, size_t uAlignment) { PRTHEAPSIMPLEBLOCK pRet = NULL; PRTHEAPSIMPLEFREE pFree; #ifdef RTHEAPSIMPLE_STRICT rtHeapSimpleAssertAll(pHeapInt); #endif /* * Search for a fitting block from the lower end of the heap. */ for (pFree = pHeapInt->pFreeHead; pFree; pFree = pFree->pNext) { uintptr_t offAlign; ASSERT_BLOCK_FREE(pHeapInt, pFree); /* * Match for size and alignment. */ if (pFree->cb < cb) continue; offAlign = (uintptr_t)(&pFree->Core + 1) & (uAlignment - 1); if (offAlign) { RTHEAPSIMPLEFREE Free; PRTHEAPSIMPLEBLOCK pPrev; offAlign = uAlignment - offAlign; if (pFree->cb - offAlign < cb) continue; /* * Make a stack copy of the free block header and adjust the pointer. */ Free = *pFree; pFree = (PRTHEAPSIMPLEFREE)((uintptr_t)pFree + offAlign); /* * Donate offAlign bytes to the node in front of us. * If we're the head node, we'll have to create a fake node. We'll * mark it USED for simplicity. * * (Should this policy of donating memory to the guy in front of us * cause big 'leaks', we could create a new free node if there is room * for that.) */ pPrev = Free.Core.pPrev; if (pPrev) { AssertMsg(!RTHEAPSIMPLEBLOCK_IS_FREE(pPrev), ("Impossible!\n")); pPrev->pNext = &pFree->Core; } else { pPrev = (PRTHEAPSIMPLEBLOCK)(pHeapInt + 1); Assert(pPrev == &pFree->Core); pPrev->pPrev = NULL; pPrev->pNext = &pFree->Core; pPrev->pHeap = pHeapInt; pPrev->fFlags = RTHEAPSIMPLEBLOCK_FLAGS_MAGIC; } pHeapInt->cbFree -= offAlign; /* * Recreate pFree in the new position and adjust the neighbors. */ *pFree = Free; /* the core */ if (pFree->Core.pNext) pFree->Core.pNext->pPrev = &pFree->Core; pFree->Core.pPrev = pPrev; /* the free part */ pFree->cb -= offAlign; if (pFree->pNext) pFree->pNext->pPrev = pFree; else pHeapInt->pFreeTail = pFree; if (pFree->pPrev) pFree->pPrev->pNext = pFree; else pHeapInt->pFreeHead = pFree; ASSERT_BLOCK_FREE(pHeapInt, pFree); ASSERT_BLOCK_USED(pHeapInt, pPrev); } /* * Split off a new FREE block? */ if (pFree->cb >= cb + RT_ALIGN_Z(sizeof(RTHEAPSIMPLEFREE), RTHEAPSIMPLE_ALIGNMENT)) { /* * Move the FREE block up to make room for the new USED block. */ PRTHEAPSIMPLEFREE pNew = (PRTHEAPSIMPLEFREE)((uintptr_t)&pFree->Core + cb + sizeof(RTHEAPSIMPLEBLOCK)); pNew->Core.pNext = pFree->Core.pNext; if (pFree->Core.pNext) pFree->Core.pNext->pPrev = &pNew->Core; pNew->Core.pPrev = &pFree->Core; pNew->Core.pHeap = pHeapInt; pNew->Core.fFlags = RTHEAPSIMPLEBLOCK_FLAGS_MAGIC | RTHEAPSIMPLEBLOCK_FLAGS_FREE; pNew->pNext = pFree->pNext; if (pNew->pNext) pNew->pNext->pPrev = pNew; else pHeapInt->pFreeTail = pNew; pNew->pPrev = pFree->pPrev; if (pNew->pPrev) pNew->pPrev->pNext = pNew; else pHeapInt->pFreeHead = pNew; pNew->cb = (pNew->Core.pNext ? (uintptr_t)pNew->Core.pNext : (uintptr_t)pHeapInt->pvEnd) \ - (uintptr_t)pNew - sizeof(RTHEAPSIMPLEBLOCK); ASSERT_BLOCK_FREE(pHeapInt, pNew); /* * Update the old FREE node making it a USED node. */ pFree->Core.fFlags &= ~RTHEAPSIMPLEBLOCK_FLAGS_FREE; pFree->Core.pNext = &pNew->Core; pHeapInt->cbFree -= pFree->cb; pHeapInt->cbFree += pNew->cb; pRet = &pFree->Core; ASSERT_BLOCK_USED(pHeapInt, pRet); } else { /* * Link it out of the free list. */ if (pFree->pNext) pFree->pNext->pPrev = pFree->pPrev; else pHeapInt->pFreeTail = pFree->pPrev; if (pFree->pPrev) pFree->pPrev->pNext = pFree->pNext; else pHeapInt->pFreeHead = pFree->pNext; /* * Convert it to a used block. */ pHeapInt->cbFree -= pFree->cb; pFree->Core.fFlags &= ~RTHEAPSIMPLEBLOCK_FLAGS_FREE; pRet = &pFree->Core; ASSERT_BLOCK_USED(pHeapInt, pRet); } break; } #ifdef RTHEAPSIMPLE_STRICT rtHeapSimpleAssertAll(pHeapInt); #endif return pRet; }