void IOBuf::reserveSlow(uint64_t minHeadroom, uint64_t minTailroom) {
size_t newCapacity = (size_t)length_ + minHeadroom + minTailroom;
DCHECK_LT(newCapacity, UINT32_MAX);
// reserveSlow() is dangerous if anyone else is sharing the buffer, as we may
// reallocate and free the original buffer. It should only ever be called if
// we are the only user of the buffer.
DCHECK(!isSharedOne());
// We'll need to reallocate the buffer.
// There are a few options.
// - If we have enough total room, move the data around in the buffer
// and adjust the data_ pointer.
// - If we're using an internal buffer, we'll switch to an external
// buffer with enough headroom and tailroom.
// - If we have enough headroom (headroom() >= minHeadroom) but not too much
// (so we don't waste memory), we can try one of two things, depending on
// whether we use jemalloc or not:
// - If using jemalloc, we can try to expand in place, avoiding a memcpy()
// - If not using jemalloc and we don't have too much to copy,
// we'll use realloc() (note that realloc might have to copy
// headroom + data + tailroom, see smartRealloc in folly/Malloc.h)
// - Otherwise, bite the bullet and reallocate.
if (headroom() + tailroom() >= minHeadroom + minTailroom) {
uint8_t* newData = writableBuffer() + minHeadroom;
memmove(newData, data_, length_);
data_ = newData;
return;
}
size_t newAllocatedCapacity = 0;
uint8_t* newBuffer = nullptr;
uint64_t newHeadroom = 0;
uint64_t oldHeadroom = headroom();
// If we have a buffer allocated with malloc and we just need more tailroom,
// try to use realloc()/xallocx() to grow the buffer in place.
SharedInfo* info = sharedInfo();
if (info && (info->freeFn == nullptr) && length_ != 0 &&
oldHeadroom >= minHeadroom) {
size_t headSlack = oldHeadroom - minHeadroom;
newAllocatedCapacity = goodExtBufferSize(newCapacity + headSlack);
if (usingJEMalloc()) {
// We assume that tailroom is more useful and more important than
// headroom (not least because realloc / xallocx allow us to grow the
// buffer at the tail, but not at the head) So, if we have more headroom
// than we need, we consider that "wasted". We arbitrarily define "too
// much" headroom to be 25% of the capacity.
if (headSlack * 4 <= newCapacity) {
size_t allocatedCapacity = capacity() + sizeof(SharedInfo);
void* p = buf_;
if (allocatedCapacity >= jemallocMinInPlaceExpandable) {
if (xallocx(p, newAllocatedCapacity, 0, 0) == newAllocatedCapacity) {
newBuffer = static_cast<uint8_t*>(p);
newHeadroom = oldHeadroom;
}
// if xallocx failed, do nothing, fall back to malloc/memcpy/free
}
}
} else { // Not using jemalloc
size_t copySlack = capacity() - length_;
if (copySlack * 2 <= length_) {
void* p = realloc(buf_, newAllocatedCapacity);
if (UNLIKELY(p == nullptr)) {
throw std::bad_alloc();
}
newBuffer = static_cast<uint8_t*>(p);
newHeadroom = oldHeadroom;
}
}
}
// None of the previous reallocation strategies worked (or we're using
// an internal buffer). malloc/copy/free.
if (newBuffer == nullptr) {
newAllocatedCapacity = goodExtBufferSize(newCapacity);
void* p = malloc(newAllocatedCapacity);
if (UNLIKELY(p == nullptr)) {
throw std::bad_alloc();
}
newBuffer = static_cast<uint8_t*>(p);
if (length_ > 0) {
assert(data_ != nullptr);
memcpy(newBuffer + minHeadroom, data_, length_);
}
if (sharedInfo()) {
freeExtBuffer();
}
newHeadroom = minHeadroom;
}
uint64_t cap;
initExtBuffer(newBuffer, newAllocatedCapacity, &info, &cap);
if (flags() & kFlagFreeSharedInfo) {
delete sharedInfo();
}
setFlagsAndSharedInfo(0, info);
capacity_ = cap;
buf_ = newBuffer;
data_ = newBuffer + newHeadroom;
// length_ is unchanged
}
/**
* Reserve enough space in the ThreadEntry::elements for the item
* @id to fit in.
*/
void StaticMetaBase::reserve(EntryID* id) {
auto& meta = *this;
ThreadEntry* threadEntry = (*threadEntry_)();
size_t prevCapacity = threadEntry->elementsCapacity;
uint32_t idval = id->getOrAllocate(meta);
if (prevCapacity > idval) {
return;
}
// Growth factor < 2, see folly/docs/FBVector.md; + 5 to prevent
// very slow start.
size_t newCapacity = static_cast<size_t>((idval + 5) * 1.7);
assert(newCapacity > prevCapacity);
ElementWrapper* reallocated = nullptr;
// Need to grow. Note that we can't call realloc, as elements is
// still linked in meta, so another thread might access invalid memory
// after realloc succeeds. We'll copy by hand and update our ThreadEntry
// under the lock.
if (usingJEMalloc()) {
bool success = false;
size_t newByteSize = nallocx(newCapacity * sizeof(ElementWrapper), 0);
// Try to grow in place.
//
// Note that xallocx(MALLOCX_ZERO) will only zero newly allocated memory,
// even if a previous allocation allocated more than we requested.
// This is fine; we always use MALLOCX_ZERO with jemalloc and we
// always expand our allocation to the real size.
if (prevCapacity * sizeof(ElementWrapper) >= jemallocMinInPlaceExpandable) {
success =
(xallocx(threadEntry->elements, newByteSize, 0, MALLOCX_ZERO) ==
newByteSize);
}
// In-place growth failed.
if (!success) {
success =
((reallocated = static_cast<ElementWrapper*>(
mallocx(newByteSize, MALLOCX_ZERO))) != nullptr);
}
if (success) {
// Expand to real size
assert(newByteSize / sizeof(ElementWrapper) >= newCapacity);
newCapacity = newByteSize / sizeof(ElementWrapper);
} else {
throw std::bad_alloc();
}
} else { // no jemalloc
// calloc() is simpler than malloc() followed by memset(), and
// potentially faster when dealing with a lot of memory, as it can get
// already-zeroed pages from the kernel.
reallocated = static_cast<ElementWrapper*>(
calloc(newCapacity, sizeof(ElementWrapper)));
if (!reallocated) {
throw std::bad_alloc();
}
}
// Success, update the entry
{
std::lock_guard<std::mutex> g(meta.lock_);
if (prevCapacity == 0) {
meta.push_back(threadEntry);
}
if (reallocated) {
/*
* Note: we need to hold the meta lock when copying data out of
* the old vector, because some other thread might be
* destructing a ThreadLocal and writing to the elements vector
* of this thread.
*/
if (prevCapacity != 0) {
memcpy(
reallocated,
threadEntry->elements,
sizeof(*reallocated) * prevCapacity);
}
std::swap(reallocated, threadEntry->elements);
}
threadEntry->elementsCapacity = newCapacity;
}
free(reallocated);
}
ElementWrapper* StaticMetaBase::reallocate(
ThreadEntry* threadEntry,
uint32_t idval,
size_t& newCapacity) {
size_t prevCapacity = threadEntry->getElementsCapacity();
// Growth factor < 2, see folly/docs/FBVector.md; + 5 to prevent
// very slow start.
auto smallCapacity = static_cast<size_t>((idval + 5) * kSmallGrowthFactor);
auto bigCapacity = static_cast<size_t>((idval + 5) * kBigGrowthFactor);
newCapacity =
(threadEntry->meta &&
(bigCapacity <= threadEntry->meta->head_.getElementsCapacity()))
? bigCapacity
: smallCapacity;
assert(newCapacity > prevCapacity);
ElementWrapper* reallocated = nullptr;
// Need to grow. Note that we can't call realloc, as elements is
// still linked in meta, so another thread might access invalid memory
// after realloc succeeds. We'll copy by hand and update our ThreadEntry
// under the lock.
if (usingJEMalloc()) {
bool success = false;
size_t newByteSize = nallocx(newCapacity * sizeof(ElementWrapper), 0);
// Try to grow in place.
//
// Note that xallocx(MALLOCX_ZERO) will only zero newly allocated memory,
// even if a previous allocation allocated more than we requested.
// This is fine; we always use MALLOCX_ZERO with jemalloc and we
// always expand our allocation to the real size.
if (prevCapacity * sizeof(ElementWrapper) >= jemallocMinInPlaceExpandable) {
success =
(xallocx(threadEntry->elements, newByteSize, 0, MALLOCX_ZERO) ==
newByteSize);
}
// In-place growth failed.
if (!success) {
success =
((reallocated = static_cast<ElementWrapper*>(
mallocx(newByteSize, MALLOCX_ZERO))) != nullptr);
}
if (success) {
// Expand to real size
assert(newByteSize / sizeof(ElementWrapper) >= newCapacity);
newCapacity = newByteSize / sizeof(ElementWrapper);
} else {
throw std::bad_alloc();
}
} else { // no jemalloc
// calloc() is simpler than malloc() followed by memset(), and
// potentially faster when dealing with a lot of memory, as it can get
// already-zeroed pages from the kernel.
reallocated = static_cast<ElementWrapper*>(
calloc(newCapacity, sizeof(ElementWrapper)));
if (!reallocated) {
throw std::bad_alloc();
}
}
return reallocated;
}
