void* MemoryManager::mallocSmallSizeSlow(uint32_t bytes, unsigned index) {
size_t nbytes = smallIndex2Size(index);
static constexpr unsigned nContigTab[] = {
#define SMALL_SIZE(index, lg_grp, lg_delta, ndelta, lg_delta_lookup, ncontig) \
ncontig,
SMALL_SIZES
#undef SMALL_SIZE
};
unsigned nContig = nContigTab[index];
size_t contigMin = nContig * nbytes;
unsigned contigInd = smallSize2Index(contigMin);
for (unsigned i = contigInd; i < kNumSmallSizes; ++i) {
FTRACE(4, "MemoryManager::mallocSmallSizeSlow({}-->{}, {}): contigMin={}, "
"contigInd={}, try i={}\n", bytes, nbytes, index, contigMin,
contigInd, i);
void* p = m_freelists[i].maybePop();
if (p != nullptr) {
FTRACE(4, "MemoryManager::mallocSmallSizeSlow({}-->{}, {}): "
"contigMin={}, contigInd={}, use i={}, size={}, p={}\n", bytes,
nbytes, index, contigMin, contigInd, i, smallIndex2Size(i),
p);
// Split tail into preallocations and store them back into freelists.
uint32_t availBytes = smallIndex2Size(i);
uint32_t tailBytes = availBytes - nbytes;
if (tailBytes > 0) {
void* tail = (void*)(uintptr_t(p) + nbytes);
splitTail(tail, tailBytes, nContig - 1, nbytes, index);
}
return p;
}
}
// No available free list items; carve new space from the current slab.
return slabAlloc(bytes, index);
}
// turn free blocks into holes, leave freelists empty.
void MemoryManager::quarantine() {
for (auto i = 0; i < kNumSmallSizes; i++) {
auto size = smallIndex2Size(i);
while (auto n = m_freelists[i].maybePop()) {
memset(n, 0x8a, size);
static_cast<FreeNode*>(n)->hdr.init(HeaderKind::Hole, size);
}
}
}
// initialize the FreeNode header on all freelist entries.
void MemoryManager::initFree() {
initHole();
for (auto i = 0; i < kNumSmallSizes; i++) {
for (auto n = m_freelists[i].head; n; n = n->next) {
n->hdr.init(HeaderKind::Free, smallIndex2Size(i));
}
}
m_needInitFree = false;
}
/*
* Allocate `bytes' from the current slab, aligned to kSmallSizeAlign.
*/
inline void* MemoryManager::slabAlloc(uint32_t bytes, unsigned index) {
FTRACE(3, "slabAlloc({}, {}): m_front={}, m_limit={}\n", bytes, index,
m_front, m_limit);
uint32_t nbytes = smallIndex2Size(index);
assert(bytes <= nbytes);
assert(nbytes <= kSlabSize);
assert((nbytes & kSmallSizeAlignMask) == 0);
assert((uintptr_t(m_front) & kSmallSizeAlignMask) == 0);
if (UNLIKELY(m_bypassSlabAlloc)) {
// Stats correction; mallocBigSize() pulls stats from jemalloc.
m_stats.usage -= bytes;
return mallocBigSize<false>(nbytes).ptr;
}
void* ptr = m_front;
{
void* next = (void*)(uintptr_t(ptr) + nbytes);
if (uintptr_t(next) <= uintptr_t(m_limit)) {
m_front = next;
} else {
ptr = newSlab(nbytes);
}
}
// Preallocate more of the same in order to amortize entry into this method.
unsigned nPrealloc;
if (nbytes * kSmallPreallocCountLimit <= kSmallPreallocBytesLimit) {
nPrealloc = kSmallPreallocCountLimit;
} else {
nPrealloc = kSmallPreallocBytesLimit / nbytes;
}
{
void* front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes);
if (uintptr_t(front) > uintptr_t(m_limit)) {
nPrealloc = ((uintptr_t)m_limit - uintptr_t(m_front)) / nbytes;
front = (void*)(uintptr_t(m_front) + nPrealloc*nbytes);
}
m_front = front;
}
for (void* p = (void*)(uintptr_t(m_front) - nbytes); p != ptr;
p = (void*)(uintptr_t(p) - nbytes)) {
m_freelists[index].push(p, nbytes);
}
FTRACE(4, "slabAlloc({}, {}) --> ptr={}, m_front={}, m_limit={}\n", bytes,
index, ptr, m_front, m_limit);
return ptr;
}
/*
* Store slab tail bytes (if any) in freelists.
*/
inline void MemoryManager::storeTail(void* tail, uint32_t tailBytes) {
void* rem = tail;
for (uint32_t remBytes = tailBytes; remBytes > 0;) {
uint32_t fragBytes = remBytes;
assert(fragBytes >= kSmallSizeAlign);
assert((fragBytes & kSmallSizeAlignMask) == 0);
unsigned fragInd = smallSize2Index(fragBytes + 1) - 1;
uint32_t fragUsable = smallIndex2Size(fragInd);
void* frag = (void*)(uintptr_t(rem) + remBytes - fragUsable);
FTRACE(4, "MemoryManager::storeTail({}, {}): rem={}, remBytes={}, "
"frag={}, fragBytes={}, fragUsable={}, fragInd={}\n", tail,
(void*)uintptr_t(tailBytes), rem, (void*)uintptr_t(remBytes),
frag, (void*)uintptr_t(fragBytes), (void*)uintptr_t(fragUsable),
fragInd);
m_freelists[fragInd].push(frag, fragUsable);
remBytes -= fragUsable;
}
}
/*
* Allocate `bytes' from the current slab, aligned to kSmallSizeAlign.
*/
inline void* MemoryManager::slabAlloc(uint32_t bytes, unsigned index) {
FTRACE(3, "slabAlloc({}, {}): m_front={}, m_limit={}\n", bytes, index,
m_front, m_limit);
uint32_t nbytes = smallIndex2Size(index);
assert(bytes <= nbytes);
assert(nbytes <= kSlabSize);
assert((nbytes & kSmallSizeAlignMask) == 0);
assert((uintptr_t(m_front) & kSmallSizeAlignMask) == 0);
if (UNLIKELY(m_bypassSlabAlloc)) {
// Stats correction; mallocBigSize() pulls stats from jemalloc.
m_stats.usage -= bytes;
return mallocBigSize<false>(nbytes).ptr;
}
void* ptr = m_front;
{
void* next = (void*)(uintptr_t(ptr) + nbytes);
if (uintptr_t(next) <= uintptr_t(m_limit)) {
m_front = next;
} else {
ptr = newSlab(nbytes);
}
}
// Preallocate more of the same in order to amortize entry into this method.
unsigned nSplit = kNContigTab[index] - 1;
uintptr_t avail = uintptr_t(m_limit) - uintptr_t(m_front);
if (UNLIKELY(nSplit * nbytes > avail)) {
nSplit = avail / nbytes; // Expensive division.
}
if (nSplit > 0) {
void* tail = m_front;
uint32_t tailBytes = nSplit * nbytes;
m_front = (void*)(uintptr_t(m_front) + tailBytes);
splitTail(tail, tailBytes, nSplit, nbytes, index);
}
FTRACE(4, "slabAlloc({}, {}) --> ptr={}, m_front={}, m_limit={}\n", bytes,
index, ptr, m_front, m_limit);
return ptr;
}
