void* MemoryManager::mallocSmallSizeSlow(uint32_t bytes, unsigned index) { size_t nbytes = smallIndex2Size(index); static constexpr unsigned nContigTab[] = { #define SMALL_SIZE(index, lg_grp, lg_delta, ndelta, lg_delta_lookup, ncontig) \ ncontig, SMALL_SIZES #undef SMALL_SIZE }; unsigned nContig = nContigTab[index]; size_t contigMin = nContig * nbytes; unsigned contigInd = smallSize2Index(contigMin); for (unsigned i = contigInd; i < kNumSmallSizes; ++i) { FTRACE(4, "MemoryManager::mallocSmallSizeSlow({}-->{}, {}): contigMin={}, " "contigInd={}, try i={}\n", bytes, nbytes, index, contigMin, contigInd, i); void* p = m_freelists[i].maybePop(); if (p != nullptr) { FTRACE(4, "MemoryManager::mallocSmallSizeSlow({}-->{}, {}): " "contigMin={}, contigInd={}, use i={}, size={}, p={}\n", bytes, nbytes, index, contigMin, contigInd, i, smallIndex2Size(i), p); // Split tail into preallocations and store them back into freelists. uint32_t availBytes = smallIndex2Size(i); uint32_t tailBytes = availBytes - nbytes; if (tailBytes > 0) { void* tail = (void*)(uintptr_t(p) + nbytes); splitTail(tail, tailBytes, nContig - 1, nbytes, index); } return p; } } // No available free list items; carve new space from the current slab. return slabAlloc(bytes, index); }
// turn free blocks into holes, leave freelists empty. void MemoryManager::quarantine() { for (auto i = 0; i < kNumSmallSizes; i++) { auto size = smallIndex2Size(i); while (auto n = m_freelists[i].maybePop()) { memset(n, 0x8a, size); static_cast<FreeNode*>(n)->hdr.init(HeaderKind::Hole, size); } } }
// initialize the FreeNode header on all freelist entries. void MemoryManager::initFree() { initHole(); for (auto i = 0; i < kNumSmallSizes; i++) { for (auto n = m_freelists[i].head; n; n = n->next) { n->hdr.init(HeaderKind::Free, smallIndex2Size(i)); } } m_needInitFree = false; }
/* * Allocate `bytes' from the current slab, aligned to kSmallSizeAlign. */ inline void* MemoryManager::slabAlloc(uint32_t bytes, unsigned index) { FTRACE(3, "slabAlloc({}, {}): m_front={}, m_limit={}\n", bytes, index, m_front, m_limit); uint32_t nbytes = smallIndex2Size(index); assert(bytes <= nbytes); assert(nbytes <= kSlabSize); assert((nbytes & kSmallSizeAlignMask) == 0); assert((uintptr_t(m_front) & kSmallSizeAlignMask) == 0); if (UNLIKELY(m_bypassSlabAlloc)) { // Stats correction; mallocBigSize() pulls stats from jemalloc. m_stats.usage -= bytes; return mallocBigSize<false>(nbytes).ptr; } void* ptr = m_front; { void* next = (void*)(uintptr_t(ptr) + nbytes); if (uintptr_t(next) <= uintptr_t(m_limit)) { m_front = next; } else { ptr = newSlab(nbytes); } } // Preallocate more of the same in order to amortize entry into this method. unsigned nPrealloc; if (nbytes * kSmallPreallocCountLimit <= kSmallPreallocBytesLimit) { nPrealloc = kSmallPreallocCountLimit; } else { nPrealloc = kSmallPreallocBytesLimit / nbytes; } { void* front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); if (uintptr_t(front) > uintptr_t(m_limit)) { nPrealloc = ((uintptr_t)m_limit - uintptr_t(m_front)) / nbytes; front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); } m_front = front; } for (void* p = (void*)(uintptr_t(m_front) - nbytes); p != ptr; p = (void*)(uintptr_t(p) - nbytes)) { m_freelists[index].push(p, nbytes); } FTRACE(4, "slabAlloc({}, {}) --> ptr={}, m_front={}, m_limit={}\n", bytes, index, ptr, m_front, m_limit); return ptr; }
/* * Store slab tail bytes (if any) in freelists. */ inline void MemoryManager::storeTail(void* tail, uint32_t tailBytes) { void* rem = tail; for (uint32_t remBytes = tailBytes; remBytes > 0;) { uint32_t fragBytes = remBytes; assert(fragBytes >= kSmallSizeAlign); assert((fragBytes & kSmallSizeAlignMask) == 0); unsigned fragInd = smallSize2Index(fragBytes + 1) - 1; uint32_t fragUsable = smallIndex2Size(fragInd); void* frag = (void*)(uintptr_t(rem) + remBytes - fragUsable); FTRACE(4, "MemoryManager::storeTail({}, {}): rem={}, remBytes={}, " "frag={}, fragBytes={}, fragUsable={}, fragInd={}\n", tail, (void*)uintptr_t(tailBytes), rem, (void*)uintptr_t(remBytes), frag, (void*)uintptr_t(fragBytes), (void*)uintptr_t(fragUsable), fragInd); m_freelists[fragInd].push(frag, fragUsable); remBytes -= fragUsable; } }
/* * Allocate `bytes' from the current slab, aligned to kSmallSizeAlign. */ inline void* MemoryManager::slabAlloc(uint32_t bytes, unsigned index) { FTRACE(3, "slabAlloc({}, {}): m_front={}, m_limit={}\n", bytes, index, m_front, m_limit); uint32_t nbytes = smallIndex2Size(index); assert(bytes <= nbytes); assert(nbytes <= kSlabSize); assert((nbytes & kSmallSizeAlignMask) == 0); assert((uintptr_t(m_front) & kSmallSizeAlignMask) == 0); if (UNLIKELY(m_bypassSlabAlloc)) { // Stats correction; mallocBigSize() pulls stats from jemalloc. m_stats.usage -= bytes; return mallocBigSize<false>(nbytes).ptr; } void* ptr = m_front; { void* next = (void*)(uintptr_t(ptr) + nbytes); if (uintptr_t(next) <= uintptr_t(m_limit)) { m_front = next; } else { ptr = newSlab(nbytes); } } // Preallocate more of the same in order to amortize entry into this method. unsigned nSplit = kNContigTab[index] - 1; uintptr_t avail = uintptr_t(m_limit) - uintptr_t(m_front); if (UNLIKELY(nSplit * nbytes > avail)) { nSplit = avail / nbytes; // Expensive division. } if (nSplit > 0) { void* tail = m_front; uint32_t tailBytes = nSplit * nbytes; m_front = (void*)(uintptr_t(m_front) + tailBytes); splitTail(tail, tailBytes, nSplit, nbytes, index); } FTRACE(4, "slabAlloc({}, {}) --> ptr={}, m_front={}, m_limit={}\n", bytes, index, ptr, m_front, m_limit); return ptr; }