fbstring IOBuf::moveToFbString() {
// malloc-allocated buffers are just fine, everything else needs
// to be turned into one.
if (!sharedInfo() || // user owned, not ours to give up
sharedInfo()->freeFn || // not malloc()-ed
headroom() != 0 || // malloc()-ed block doesn't start at beginning
tailroom() == 0 || // no room for NUL terminator
isShared() || // shared
isChained()) { // chained
// We might as well get rid of all head and tailroom if we're going
// to reallocate; we need 1 byte for NUL terminator.
coalesceAndReallocate(0, computeChainDataLength(), this, 1);
}
// Ensure NUL terminated
*writableTail() = 0;
fbstring str(reinterpret_cast<char*>(writableData()),
length(), capacity(),
AcquireMallocatedString());
if (flags() & kFlagFreeSharedInfo) {
delete sharedInfo();
}
// Reset to a state where we can be deleted cleanly
flagsAndSharedInfo_ = 0;
buf_ = nullptr;
clear();
return str;
}
void IOBuf::coalesceSlow(size_t maxLength) {
// coalesceSlow() should only be called if we are part of a chain of multiple
// IOBufs. The caller should have already verified this.
DCHECK(isChained());
DCHECK_LT(length_, maxLength);
// Compute the length of the entire chain
uint64_t newLength = 0;
IOBuf* end = this;
while (true) {
newLength += end->length_;
end = end->next_;
if (newLength >= maxLength) {
break;
}
if (end == this) {
throw std::overflow_error("attempted to coalesce more data than "
"available");
}
}
coalesceAndReallocate(newLength, end);
// We should have the requested length now
DCHECK_GE(length_, maxLength);
}
IOBuf IOBuf::cloneCoalescedAsValue() const {
if (!isChained()) {
return cloneOneAsValue();
}
// Coalesce into newBuf
const uint64_t newLength = computeChainDataLength();
const uint64_t newHeadroom = headroom();
const uint64_t newTailroom = prev()->tailroom();
const uint64_t newCapacity = newLength + newHeadroom + newTailroom;
IOBuf newBuf{CREATE, newCapacity};
newBuf.advance(newHeadroom);
auto current = this;
do {
if (current->length() > 0) {
DCHECK_NOTNULL(current->data());
DCHECK_LE(current->length(), newBuf.tailroom());
memcpy(newBuf.writableTail(), current->data(), current->length());
newBuf.append(current->length());
}
current = current->next();
} while (current != this);
DCHECK_EQ(newLength, newBuf.length());
DCHECK_EQ(newHeadroom, newBuf.headroom());
DCHECK_LE(newTailroom, newBuf.tailroom());
return newBuf;
}
void IOBuf::coalesceSlow() {
// coalesceSlow() should only be called if we are part of a chain of multiple
// IOBufs. The caller should have already verified this.
DCHECK(isChained());
// Compute the length of the entire chain
uint64_t newLength = 0;
IOBuf* end = this;
do {
newLength += end->length_;
end = end->next_;
} while (end != this);
coalesceAndReallocate(newLength, end);
// We should be only element left in the chain now
DCHECK(!isChained());
}
void IOBuf::makeManagedChained() {
assert(isChained());
IOBuf* current = this;
while (true) {
current->makeManagedOne();
current = current->next_;
if (current == this) {
break;
}
}
}
void IOBuf::coalesceAndReallocate(size_t newHeadroom,
size_t newLength,
IOBuf* end,
size_t newTailroom) {
uint64_t newCapacity = newLength + newHeadroom + newTailroom;
if (newCapacity > UINT32_MAX) {
throw std::overflow_error("IOBuf chain too large to coalesce");
}
// Allocate space for the coalesced buffer.
// We always convert to an external buffer, even if we happened to be an
// internal buffer before.
uint8_t* newBuf;
SharedInfo* newInfo;
uint32_t actualCapacity;
allocExtBuffer(newCapacity, &newBuf, &newInfo, &actualCapacity);
// Copy the data into the new buffer
uint8_t* newData = newBuf + newHeadroom;
uint8_t* p = newData;
IOBuf* current = this;
size_t remaining = newLength;
do {
assert(current->length_ <= remaining);
remaining -= current->length_;
memcpy(p, current->data_, current->length_);
p += current->length_;
current = current->next_;
} while (current != end);
assert(remaining == 0);
// Point at the new buffer
decrementRefcount();
// Make sure kFlagUserOwned, kFlagMaybeShared, and kFlagFreeSharedInfo
// are all cleared.
flags_ = 0;
capacity_ = actualCapacity;
type_ = kExtAllocated;
buf_ = newBuf;
sharedInfo_ = newInfo;
data_ = newData;
length_ = newLength;
// Separate from the rest of our chain.
// Since we don't store the unique_ptr returned by separateChain(),
// this will immediately delete the returned subchain.
if (isChained()) {
(void)separateChain(next_, current->prev_);
}
}
void IOBuf::coalesceAndReallocate(size_t newHeadroom,
size_t newLength,
IOBuf* end,
size_t newTailroom) {
uint64_t newCapacity = newLength + newHeadroom + newTailroom;
// Allocate space for the coalesced buffer.
// We always convert to an external buffer, even if we happened to be an
// internal buffer before.
uint8_t* newBuf;
SharedInfo* newInfo;
uint64_t actualCapacity;
allocExtBuffer(newCapacity, &newBuf, &newInfo, &actualCapacity);
// Copy the data into the new buffer
uint8_t* newData = newBuf + newHeadroom;
uint8_t* p = newData;
IOBuf* current = this;
size_t remaining = newLength;
do {
if (current->length_ > 0) {
assert(current->length_ <= remaining);
assert(current->data_ != nullptr);
remaining -= current->length_;
memcpy(p, current->data_, current->length_);
p += current->length_;
}
current = current->next_;
} while (current != end);
assert(remaining == 0);
// Point at the new buffer
decrementRefcount();
// Make sure kFlagMaybeShared and kFlagFreeSharedInfo are all cleared.
setFlagsAndSharedInfo(0, newInfo);
capacity_ = actualCapacity;
buf_ = newBuf;
data_ = newData;
length_ = newLength;
// Separate from the rest of our chain.
// Since we don't store the unique_ptr returned by separateChain(),
// this will immediately delete the returned subchain.
if (isChained()) {
(void)separateChain(next_, current->prev_);
}
}
void IOBuf::unshareChained() {
// unshareChained() should only be called if we are part of a chain of
// multiple IOBufs. The caller should have already verified this.
assert(isChained());
IOBuf* current = this;
while (true) {
if (current->isSharedOne()) {
// we have to unshare
break;
}
current = current->next_;
if (current == this) {
// None of the IOBufs in the chain are shared,
// so return without doing anything
return;
}
}
// We have to unshare. Let coalesceSlow() do the work.
coalesceSlow();
}
