/* * Allocate `bytes' from the current slab, aligned to kSmartSizeAlign. */ void* MemoryManager::slabAlloc(uint32_t bytes, unsigned index) { FTRACE(3, "slabAlloc({}, {})\n", bytes, index); size_t nbytes = debugAddExtra(smartSizeClass(bytes)); assert(nbytes <= kSlabSize); assert((nbytes & kSmartSizeAlignMask) == 0); assert((uintptr_t(m_front) & kSmartSizeAlignMask) == 0); if (UNLIKELY(m_bypassSlabAlloc)) { // Stats correction; smartMallocSizeBig() pulls stats from jemalloc. m_stats.usage -= bytes; // smartMallocSizeBig already wraps its allocation in a debug header, but // the caller will try to do it again, so we need to adjust this pointer // before returning it. return ((char*)smartMallocSizeBig<false>(nbytes).ptr) - kDebugExtraSize; } void* ptr = m_front; { void* next = (void*)(uintptr_t(ptr) + nbytes); if (uintptr_t(next) <= uintptr_t(m_limit)) { m_front = next; } else { ptr = newSlab(nbytes); } } // Preallocate more of the same in order to amortize entry into this method. unsigned nPrealloc; if (nbytes * kSmartPreallocCountLimit <= kSmartPreallocBytesLimit) { nPrealloc = kSmartPreallocCountLimit; } else { nPrealloc = kSmartPreallocBytesLimit / nbytes; } { void* front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); if (uintptr_t(front) > uintptr_t(m_limit)) { nPrealloc = ((uintptr_t)m_limit - uintptr_t(m_front)) / nbytes; front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); } m_front = front; } for (void* p = (void*)(uintptr_t(m_front) - nbytes); p != ptr; p = (void*)(uintptr_t(p) - nbytes)) { auto usable = debugRemoveExtra(nbytes); auto ptr = debugPostAllocate(p, usable, usable); debugPreFree(ptr, usable, usable); m_freelists[index].push(ptr, usable); } return ptr; }
/* * Allocate `bytes' from the current slab, aligned to kSmartSizeAlign. */ inline void* MemoryManager::slabAlloc(uint32_t bytes, unsigned index) { FTRACE(3, "slabAlloc({}, {}): m_front={}, m_limit={}\n", bytes, index, m_front, m_limit); uint32_t nbytes = smartSizeClass(bytes); assert(nbytes <= kSlabSize); assert((nbytes & kSmartSizeAlignMask) == 0); assert((uintptr_t(m_front) & kSmartSizeAlignMask) == 0); if (UNLIKELY(m_bypassSlabAlloc)) { // Stats correction; smartMallocSizeBig() pulls stats from jemalloc. m_stats.usage -= bytes; return smartMallocSizeBig<false>(nbytes).ptr; } void* ptr = m_front; { void* next = (void*)(uintptr_t(ptr) + nbytes); if (uintptr_t(next) <= uintptr_t(m_limit)) { m_front = next; } else { ptr = newSlab(nbytes); } } // Preallocate more of the same in order to amortize entry into this method. unsigned nPrealloc; if (nbytes * kSmartPreallocCountLimit <= kSmartPreallocBytesLimit) { nPrealloc = kSmartPreallocCountLimit; } else { nPrealloc = kSmartPreallocBytesLimit / nbytes; } { void* front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); if (uintptr_t(front) > uintptr_t(m_limit)) { nPrealloc = ((uintptr_t)m_limit - uintptr_t(m_front)) / nbytes; front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); } m_front = front; } for (void* p = (void*)(uintptr_t(m_front) - nbytes); p != ptr; p = (void*)(uintptr_t(p) - nbytes)) { m_freelists[index].push(p, nbytes); } FTRACE(4, "slabAlloc({}, {}) --> ptr={}, m_front={}, m_limit={}\n", bytes, index, ptr, m_front, m_limit); return ptr; }
/* * Allocate `bytes' from the current slab, aligned to kSmartSizeAlign. */ void* MemoryManager::slabAlloc(uint32_t bytes, unsigned index) { size_t nbytes = debugAddExtra(smartSizeClass(bytes)); assert(nbytes <= kSlabSize); assert((nbytes & kSmartSizeAlignMask) == 0); assert((uintptr_t(m_front) & kSmartSizeAlignMask) == 0); if (UNLIKELY(m_profctx.flag)) { // Stats correction; smartMallocSizeBig() pulls stats from jemalloc. m_stats.usage -= bytes; return smartMallocSizeBig<false>(nbytes).first; } void* ptr = m_front; { void* next = (void*)(uintptr_t(ptr) + nbytes); if (uintptr_t(next) <= uintptr_t(m_limit)) { m_front = next; } else { ptr = newSlab(nbytes); } } // Preallocate more of the same in order to amortize entry into this method. unsigned nPrealloc; if (nbytes * kSmartPreallocCountLimit <= kSmartPreallocBytesLimit) { nPrealloc = kSmartPreallocCountLimit; } else { nPrealloc = kSmartPreallocBytesLimit / nbytes; } { void* front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); if (uintptr_t(front) > uintptr_t(m_limit)) { nPrealloc = ((uintptr_t)m_limit - uintptr_t(m_front)) / nbytes; front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); } m_front = front; } for (void* p = (void*)(uintptr_t(m_front) - nbytes); p != ptr; p = (void*)(uintptr_t(p) - nbytes)) { m_freelists[index].push( debugPreFree(debugPostAllocate(p, debugRemoveExtra(nbytes), debugRemoveExtra(nbytes)), debugRemoveExtra(nbytes), debugRemoveExtra(nbytes))); } return ptr; }
inline void* MemoryManager::smartMalloc(size_t nbytes) { nbytes += sizeof(SmallNode); if (UNLIKELY(nbytes > kMaxSmartSize)) { return smartMallocBig(nbytes); } nbytes = smartSizeClass(nbytes); m_stats.usage += nbytes; auto const idx = (nbytes - 1) >> kLgSizeQuantum; assert(idx < kNumSizes && idx >= 0); void* vp = m_sizeTrackedFree[idx].maybePop(); if (UNLIKELY(vp == nullptr)) { return smartMallocSlab(nbytes); } FTRACE(1, "smartMalloc: {} -> {}\n", nbytes, vp); return vp; }