inline void* MemoryManager::smartRealloc(void* inputPtr, size_t nbytes) {
FTRACE(1, "smartRealloc: {} to {}\n", inputPtr, nbytes);
assert(nbytes > 0);
void* ptr = debug ? static_cast<DebugHeader*>(inputPtr) - 1 : inputPtr;
auto const n = static_cast<SweepNode*>(ptr) - 1;
if (LIKELY(n->padbytes <= kMaxSmartSize)) {
void* newmem = smart_malloc(nbytes);
auto const copySize = std::min(
n->padbytes - sizeof(SmallNode) - (debug ? sizeof(DebugHeader) : 0),
nbytes
);
newmem = memcpy(newmem, inputPtr, copySize);
smart_free(inputPtr);
return newmem;
}
// Ok, it's a big allocation. Since we don't know how big it is
// (i.e. how much data we should memcpy), we have no choice but to
// ask malloc to realloc for us.
auto const oldNext = n->next;
auto const oldPrev = n->prev;
auto const newNode = static_cast<SweepNode*>(
realloc(n, debugAddExtra(nbytes + sizeof(SweepNode)))
);
refreshStatsHelper();
if (newNode != n) {
oldNext->prev = oldPrev->next = newNode;
}
return debugPostAllocate(newNode + 1, 0, 0);
}
inline void* MemoryManager::smartEnlist(SweepNode* n) {
if (UNLIKELY(m_stats.usage > m_stats.maxBytes)) {
refreshStatsHelper();
}
// link after m_sweep
SweepNode* next = m_sweep.next;
n->next = next;
n->prev = &m_sweep;
next->prev = m_sweep.next = n;
assert(n->padbytes > kMaxSmartSize);
return n + 1;
}
/*
* Get a new slab, then allocate nbytes from it and install it in our
* slab list. Return the newly allocated nbytes-sized block.
*/
NEVER_INLINE void* MemoryManager::newSlab(size_t nbytes) {
if (UNLIKELY(m_stats.usage > m_stats.maxBytes)) {
refreshStatsHelper();
}
void* slab = safe_malloc(kSlabSize);
assert((uintptr_t(slab) & kSmartSizeAlignMask) == 0);
JEMALLOC_STATS_ADJUST(&m_stats, kSlabSize);
m_stats.alloc += kSlabSize;
if (m_stats.alloc > m_stats.peakAlloc) {
m_stats.peakAlloc = m_stats.alloc;
}
m_slabs.push_back(slab);
m_front = (void*)(uintptr_t(slab) + nbytes);
m_limit = (void*)(uintptr_t(slab) + kSlabSize);
FTRACE(3, "newSlab: adding slab at {} to limit {}\n", slab, m_limit);
return slab;
}
/*
* Get a new slab, then allocate nbytes from it and install it in our
* slab list. Return the newly allocated nbytes-sized block.
*/
NEVER_INLINE char* MemoryManager::newSlab(size_t nbytes) {
if (UNLIKELY(m_stats.usage > m_stats.maxBytes)) {
refreshStatsHelper();
}
char* slab = (char*) Util::safe_malloc(SLAB_SIZE);
assert(uintptr_t(slab) % 16 == 0);
JEMALLOC_STATS_ADJUST(&m_stats, SLAB_SIZE);
m_stats.alloc += SLAB_SIZE;
if (m_stats.alloc > m_stats.peakAlloc) {
m_stats.peakAlloc = m_stats.alloc;
}
m_slabs.push_back(slab);
m_front = slab + nbytes;
m_limit = slab + SLAB_SIZE;
FTRACE(1, "newSlab: adding slab at {} to limit {}\n",
static_cast<void*>(slab),
static_cast<void*>(m_limit));
return slab;
}
