/* * Allocate `bytes' from the current slab, aligned to kSmartSizeAlign. */ void* MemoryManager::slabAlloc(uint32_t bytes, unsigned index) { FTRACE(3, "slabAlloc({}, {})\n", bytes, index); size_t nbytes = debugAddExtra(smartSizeClass(bytes)); assert(nbytes <= kSlabSize); assert((nbytes & kSmartSizeAlignMask) == 0); assert((uintptr_t(m_front) & kSmartSizeAlignMask) == 0); if (UNLIKELY(m_bypassSlabAlloc)) { // Stats correction; smartMallocSizeBig() pulls stats from jemalloc. m_stats.usage -= bytes; // smartMallocSizeBig already wraps its allocation in a debug header, but // the caller will try to do it again, so we need to adjust this pointer // before returning it. return ((char*)smartMallocSizeBig<false>(nbytes).ptr) - kDebugExtraSize; } void* ptr = m_front; { void* next = (void*)(uintptr_t(ptr) + nbytes); if (uintptr_t(next) <= uintptr_t(m_limit)) { m_front = next; } else { ptr = newSlab(nbytes); } } // Preallocate more of the same in order to amortize entry into this method. unsigned nPrealloc; if (nbytes * kSmartPreallocCountLimit <= kSmartPreallocBytesLimit) { nPrealloc = kSmartPreallocCountLimit; } else { nPrealloc = kSmartPreallocBytesLimit / nbytes; } { void* front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); if (uintptr_t(front) > uintptr_t(m_limit)) { nPrealloc = ((uintptr_t)m_limit - uintptr_t(m_front)) / nbytes; front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); } m_front = front; } for (void* p = (void*)(uintptr_t(m_front) - nbytes); p != ptr; p = (void*)(uintptr_t(p) - nbytes)) { auto usable = debugRemoveExtra(nbytes); auto ptr = debugPostAllocate(p, usable, usable); debugPreFree(ptr, usable, usable); m_freelists[index].push(ptr, usable); } return ptr; }
/* * Allocate `bytes' from the current slab, aligned to kSmartSizeAlign. */ void* MemoryManager::slabAlloc(uint32_t bytes, unsigned index) { size_t nbytes = debugAddExtra(smartSizeClass(bytes)); assert(nbytes <= kSlabSize); assert((nbytes & kSmartSizeAlignMask) == 0); assert((uintptr_t(m_front) & kSmartSizeAlignMask) == 0); if (UNLIKELY(m_profctx.flag)) { // Stats correction; smartMallocSizeBig() pulls stats from jemalloc. m_stats.usage -= bytes; return smartMallocSizeBig<false>(nbytes).first; } void* ptr = m_front; { void* next = (void*)(uintptr_t(ptr) + nbytes); if (uintptr_t(next) <= uintptr_t(m_limit)) { m_front = next; } else { ptr = newSlab(nbytes); } } // Preallocate more of the same in order to amortize entry into this method. unsigned nPrealloc; if (nbytes * kSmartPreallocCountLimit <= kSmartPreallocBytesLimit) { nPrealloc = kSmartPreallocCountLimit; } else { nPrealloc = kSmartPreallocBytesLimit / nbytes; } { void* front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); if (uintptr_t(front) > uintptr_t(m_limit)) { nPrealloc = ((uintptr_t)m_limit - uintptr_t(m_front)) / nbytes; front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); } m_front = front; } for (void* p = (void*)(uintptr_t(m_front) - nbytes); p != ptr; p = (void*)(uintptr_t(p) - nbytes)) { m_freelists[index].push( debugPreFree(debugPostAllocate(p, debugRemoveExtra(nbytes), debugRemoveExtra(nbytes)), debugRemoveExtra(nbytes), debugRemoveExtra(nbytes))); } return ptr; }
// allocate nbytes from the current slab, aligned to kSmartSizeAlign void* MemoryManager::slabAlloc(size_t nbytes, unsigned index) { assert(nbytes <= kSlabSize); assert((nbytes & kSmartSizeAlignMask) == 0); assert((uintptr_t(m_front) & kSmartSizeAlignMask) == 0); void* ptr = m_front; { void* next = (void*)(uintptr_t(ptr) + nbytes); if (uintptr_t(next) <= uintptr_t(m_limit)) { m_front = next; } else { ptr = newSlab(nbytes); } } // Preallocate more of the same in order to amortize entry into this method. unsigned nPrealloc; if (nbytes * kSmartPreallocCountLimit <= kSmartPreallocBytesLimit) { nPrealloc = kSmartPreallocCountLimit; } else { nPrealloc = kSmartPreallocBytesLimit / nbytes; } { void* front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); if (uintptr_t(front) > uintptr_t(m_limit)) { nPrealloc = ((uintptr_t)m_limit - uintptr_t(m_front)) / nbytes; front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes); } m_front = front; } for (void* p = (void*)(uintptr_t(m_front) - nbytes); p != ptr; p = (void*)(uintptr_t(p) - nbytes)) { m_freelists[index].push( debugPreFree(debugPostAllocate(p, debugRemoveExtra(nbytes), debugRemoveExtra(nbytes)), debugRemoveExtra(nbytes), debugRemoveExtra(nbytes))); } return ptr; }